Merge branch 'origin/unify_uwsgi_python' into 'unify_uwsgi_python'

Unify Unification Branches

See merge request !2

protokollsystem/defaults/main.yml

----
-# file: protokollsystem/defaults/main.yml
-
-protokolle_web_root: /var/www/protokollsystem
-protokolle_name: protokollsystem
-protokolle_user: protokolle
-protokolle_group: protokolle
-protokolle_celery_concurrency: 4
-protokolle_ldap_cert: ''
-protokolle_mail: True
-protokolle_mail_from: 'sender@example.com'
-protokolle_mail_host: 'mail.example.com:25'
-protokolle_mail_user: ''
-protokolle_mail_password: ''
-protokolle_mail_tls: False # 'tls' or 'starttls'
-protokolle_celery_broker: 'redis://localhost:6379/0'
-protokolle_url_root: 'protokolle.example.com'
-protokolle_url_proto: 'https'
-protokolle_url_path: '/'
-protokolle_printing: True
-protokolle_printing_server: 'printsrv.example.com:631'
-protokolle_printing_user: 'protokolle'
-protokolle_printing_printers:
-  - printer: kopierer
-    options: ["ColorModel=Gray", "KCStaple=Center", "KCPunch=2HoleEUR", "Duplex=DuplexNoTumble"]
-protokolle_etherpad: True
-protokolle_etherpad_url: 'https://pad.example.com'
-protokolle_wiki: True
-protokolle_wiki_type: MEDIAWIKI
-protokolle_wiki_api: 'https://wiki.example.com/wiki/api.php'
-protokolle_wiki_anonymous: False
-protokolle_wiki_user: ''
-protokolle_wiki_password: ''
-protokolle_wiki_domain: ''
-protokolle_calendar: True
-protokolle_calendar_url: 'https://user:password@groupware.example.com/SOGo/dav/user/Calendar/personal'
-protokolle_admin_mail: 'admin@example.com'
-protokolle_admin_group: 'protokolladmin'
-protokolle_latex_local_templates: '' # local-templates
-protokolle_latex_logo_template: '' # logo.tex
-protokolle_latex_geometry: '' # bottom=1.6cm,top=1.6cm,inner=2.5cm,outer=1.0cm,footskip=1.0cm,headsep=0.6cm
-protokolle_latex_pagestyle: '' # fancy
-protokolle_latex_packages: [] # ["[absolute]{textpos}", "{fancyheadings}"]
-protokolle_latex_header_footer: None # True
-protokolle_logos: []
-protokolle_auth_max_duration: 86400
-protokolle_auth_backends:
-  - type: ADManager
-    host: auth.example.com
-    domain: EXAMPLE
-    user_dn: 'cn=users,dc=example,dc=com'
-    group_dn: 'dc=example,dc=com'
-    ca_cert: ''
-    obsolete: False
-protokolle_auth_obsoletion_warning: ''
-

protokollsystem/handlers/main.yml

----
-# file: protokollsystem/handlers/main.yml
-
-- name: reload systemd service files
-  command: systemctl daemon-reload
-
-- name: restart uwsgi for protokollsystem
-  service:
-    name: "{{ item }}"
-    state: restarted
-  with_items:
-    - "{{ protokolle_name }}"
-    - "{{ protokolle_name }}-celery"
-
-- name: create tmpfiles
-  command: systemd-tmpfiles --create

protokollsystem/meta/main.yml

----
-# file: protokollsystem/meta/main.yml
-dependencies:
-  - { role: texlive }
-  - { role: cups-client }
-  - { role: uwsgi-python, uwsgi_name: "{{protokolle_name}}", uwsgi_user: "{{protokolle_user}}", uwsgi_group: "{{protokolle_group}}", uwsgi_path: "{{protokolle_web_root}}", uwsgi_home: "{{protokolle_web_root}}", uwsgi_program: "server.py", uwsgi_callable: "app", uwsgi_command: "runserver", uwsgi_db: "postgres", uwsgi_python: 3, uwsgi_mules: 1, uwsgi_options: ["enable-threads"] }

protokollsystem/tasks/main.yml

----
-# file: protokollsystem/tasks/main.yml
-
-- name: ensure we have all required software and fonts
-  apt:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python3-virtualenv
-    - virtualenv
-    - libxml2-dev
-    - libxslt-dev
-    - fontconfig
-    - tex-gyre
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: ensure the deploy key is available
-  copy:
-    src: "{{ protokolle_deploy_key }}"
-    dest: /root/.ssh/protokolle
-    owner: root
-    group: root
-    mode: 0600
-  tags:
-    - protokollsystem
-    - webservices
-
-# https://github.com/ansible/ansible/issues/27699
-- name: ensure fucking git module is able to clone
-  command: mount -o remount,exec /tmp
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: ensure we have the program
-  git:
-    repo: git@git.fsmpi.rwth-aachen.de:protokollsystem/proto3.git
-    dest: "{{ protokolle_web_root }}"
-    accept_hostkey: True # TODO remove this
-    key_file: /root/.ssh/protokolle
-  notify:
-    - restart uwsgi for protokollsystem
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: ensure fucking git module is not able to clone anymore
-  command: mount -o remount,noexec /tmp
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: ensure we have a virtualenv
-  pip:
-    requirements: "{{ protokolle_web_root }}/requirements.txt"
-    virtualenv: "{{ protokolle_web_root }}/"
-    virtualenv_python: python3
-  notify:
-    - restart uwsgi for protokollsystem
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: ensure we have our config
-  template:
-    src: config.py.j2
-    dest: "{{ protokolle_web_root }}/config.py"
-    owner: "{{ protokolle_user }}"
-    group: "{{ protokolle_group }}"
-    mode: 0640
-  notify:
-    - restart uwsgi for protokollsystem
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: check our config
-  command: "{{protokolle_web_root}}/bin/python {{protokolle_web_root}}/configproxy.py check --log-level warning"
-  args:
-    chdir: "{{protokolle_web_root}}"
-  become: yes
-  become_user: "{{protokolle_user}}"
-  changed_when: no
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: ensure data model upgrades are applied
-  command: "{{protokolle_web_root}}/bin/python {{protokolle_web_root}}/server.py db upgrade"
-  args:
-    chdir: "{{protokolle_web_root}}"
-  become: yes
-  become_user: "{{protokolle_user}}"
-  notify:
-    - restart uwsgi for protokollsystem
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: ensure we have our local templates
-  copy:
-    src: "{{ protokolle_local_templates }}"
-    dest: "{{ protokolle_web_root }}/"
-    owner: "{{ protokolle_user }}"
-    group: "{{ protokolle_group }}"
-    mode: 0644
-  when: protokolle_local_templates|default('') != ''
-  notify:
-    - restart uwsgi for protokollsystem
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: ensure one local template is the default
-  file:
-    src: "{{ protokolle_web_root }}/{{ protokolle_latex_local_templates }}/{{ protokolle_local_templates_default }}/{{ item.path }}"
-    dest: "{{ protokolle_web_root }}/{{ protokolle_latex_local_templates }}/{{ item.path }}"
-    state: link
-  with_filetree: "{{ protokolle_local_templates }}/{{ protokolle_local_templates_default }}"
-  when: protokolle_local_templates|default(False) and protokolle_local_templates_default|default(False)
-  notify:
-    - restart uwsgi for protokollsystem
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: ensure the unit file exists
-  template:
-    src: protokollsystem.service.j2
-    dest: "/etc/systemd/system/{{ protokolle_name }}.service"
-    owner: root
-    group: root
-    mode: 0644
-  notify:
-    - reload systemd service files
-    - restart uwsgi for protokollsystem
-  tags:
-    - protokollsystem
-    - webservices
-
-- name: ensure the celery unit file exists
-  template:
-    src: celery.service.j2
-    dest: "/etc/systemd/system/{{ protokolle_name }}-celery.service"
-    owner: root
-    group: root
-    mode: 0644
-  notify:
-    - reload systemd service files
-    - restart uwsgi for protokollsystem
-  tags:
-    - protokollsystem
-    - webservices
-
-- meta: flush_handlers
-
-- name: ensure the services are enabled
-  service:
-    name: "{{ item }}"
-    enabled: yes
-    state: started
-  with_items:
-    - "{{ protokolle_name }}"
-    - "{{ protokolle_name }}-celery"
-  tags:
-    - protokollsystem
-    - webservices

protokollsystem/templates/celery.service.j2

-[Unit]
-Description={{ protokolle_name }}-Celery
-After=network.target
-
-[Service]
-User={{ protokolle_user }}
-Group={{ protokolle_group }}
-WorkingDirectory={{ protokolle_web_root }}
-Environment=VIRTUAL_ENV="{{ protokolle_web_root }}"
-ExecStart={{ protokolle_web_root }}/bin/celery -A server.celery worker --loglevel=DEBUG --concurrency={{ protokolle_celery_concurrency }}
-Restart=always
-
-[Install]
-WantedBy=multi-user.target

protokollsystem/templates/protokollsystem.service.j2

-[Unit]
-Description={{ protokolle_name }}
-After=network.target
-Wants=protokollsystem-celery.service
-
-[Service]
-{% if protokolle_ldap_cert %}
-Environment=LDAPTLS_CACERT={{ protokolle_ldap_cert }}
-{% endif %}
-ExecStart=/usr/bin/uwsgi --ini /etc/uwsgi/apps-available/{{ protokolle_name }}.ini
-ExecReload=/bin/kill -HUP $MAINPID
-ExecStop=/bin/kill -INT $MAINPID
-Restart=always
-Type=notify
-NotifyAccess=all
-KillSignal=SIGQUIT
-
-[Install]
-WantedBy=multi-user.target

uwsgi-python/defaults/main.yml

----
-# files: uwsgi-python/defaults/main.yml
-
-uwsgi_name: uwsgi
-uwsgi_user: uwsgi
-uwsgi_group: uwsgi
-uwsgi_home: /var/www
-uwsgi_path: /var/www
-uwsgi_program: server.py
-uwsgi_callable: app
-uwsgi_command: runserver
-uwsgi_db: none
-uwsgi_python: 3
-uwsgi_mules: 0
-uwsgi_mountpoint: /
-uwsgi_enable_threads: false

uwsgi-python/handlers/main.yml

@@ -24,3 +24,11 @@
 
 - name: restart uwsgi instance migration-webapp
   service: name="uwsgi@migration-webapp" state=restarted
+
+- name: restart uwsgi instance protokollsystem uwsgi
+  service: name="uwsgi@protokollsystem" state=restarted
+  listen: "restart uwsgi instance protokollsystem"
+
+- name: restart uwsgi instance protokollsystem celery
+  service: name="protokollsystem-celery" state=restarted
+  listen: "restart uwsgi instance protokollsystem"

uwsgi-python/tasks/apps/protokollsystem.yml

+---
+# file: protokollsystem/tasks/main.yml
+
+- name: check our config
+  command: "{{app_path}/bin/python {{app_path}}/configproxy.py check --log-level warning"
+  args:
+    chdir: "{{app_path}}"
+  become: yes
+  become_user: "{{app_user}}"
+  changed_when: no
+
+- name: ensure data model upgrades are applied
+  command: "{{app_path}}/bin/python {{app_path}}/server.py db upgrade"
+  args:
+    chdir: "{{app_path}}"
+  become: yes
+  become_user: "{{app_user}}"
+  notify:
+    - "restart uwsgi instance {{app.instance}}" 
+
+- name: ensure we have our local templates
+  copy:
+    src: "{{ protokolle_local_templates }}"
+    dest: "{{ app_path }}/"
+    owner: "{{ protokolle_user }}"
+    group: "{{ protokolle_group }}"
+    mode: 0644
+  when: protokolle_local_templates|default('') != ''
+  notify:
+    - "restart uwsgi instance {{app.instance}}"
+
+- name: ensure one local template is the default
+  file:
+    src: "{{ app_path }}/{{ protokolle_latex_local_templates }}/{{ protokolle_local_templates_default }}/{{ item.path }}"
+    dest: "{{ app_path }}/{{ protokolle_latex_local_templates }}/{{ item.path }}"
+    state: link
+  with_filetree: "{{ protokolle_local_templates }}/{{ protokolle_local_templates_default }}"
+  when: protokolle_local_templates|default(False) and protokolle_local_templates_default|default(False)
+  notify:
+    - "restart uwsgi instance {{app.instance}}"
+
+- name: ensure the celery unit file exists
+  template:
+    src: apps/protokollsystem-celery.service.j2
+    dest: "/etc/systemd/system/{{ app_name }}-celery.service"
+    owner: root
+    group: root
+    mode: 0644
+  notify:
+    - reload systemd service files
+    - "restart uwsgi instance {{app.instance}}"
+
+- name: ensure the celery service is enabled
+  service:
+    name: "{{app_name}}-celery"
+    enabled: yes
+    state: started

uwsgi-python/tasks/apps/schilder.yml

+# https://github.com/ansible/ansible/issues/27699
+- name: ensure git module is able to clone
+  command: mount -o remount,exec /tmp
+
 - name: ensure we have our templates
   git:
     repo: "{{ schilder_templates_url }}"
     dest: "{{ app_home }}/tex"
-    key_file: /root/.ssh/schildergenerator
+    key_file: /root/.ssh/schilder
     version: HEAD
   notify:
     - "restart uwsgi instance {{ app.instance }}"
+
+- name: ensure git module is not able to clone anymore
+  command: mount -o remount,noexec /tmp

uwsgi-python/tasks/main.yml

@@ -38,6 +38,7 @@
   with_items: "{{ webapps }}"
   loop_control:
     loop_var: app
+    label: "{{app.app}} {{app.instance}}"
   tags:
     - uwsgi
     - webservices

uwsgi-python/templates/apps/protokollsystem-celery.service.j2

+[Unit]
+Description={{ app_name }}-Celery
+After=network.target
+
+[Service]
+User={{ app_user }}
+Group={{ app_group }}
+WorkingDirectory={{ app_path }}
+Environment=VIRTUAL_ENV="{{ app_path }}"
+ExecStart={{ app_path }}/bin/celery -A server.celery worker --loglevel=DEBUG --concurrency={{ protokolle_celery_concurrency }}
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

protokollsystem/templates/config.py.j2 → uwsgi-python/templates/apps/protokollsystem.j2

-SQLALCHEMY_DATABASE_URI = "postgresql://{{ protokolle_user }}:@/{{ protokolle_name }}"
+SQLALCHEMY_DATABASE_URI = "postgresql://{{app_user}}:@/{{app_name}}"
 SQLALCHEMY_TRACK_MODIFICATIONS = False
 
-SECRET_KEY = "{{ protokolle_secret }}"
+from secret_config import secret_key as SECRET_KEY, security_key as SECURITY_KEY
 
 DEBUG = False
 

uwsgi-python/templates/secret_config.py.j2

 secret_key = '{{ (2**2048)|random }}'
+{% if app_secret_config_keys is defined %}
+{% for key in app_secret_config_keys %}
+{{key}} = '{{ (2**2048)|random }}'
+{% endfor %}
+{% endif %}

uwsgi-python/templates/uwsgi.ini.j2

@@ -8,18 +8,18 @@ master =
 processes = 4
 workers = 4
 prio = -5
-harakiri = {{uwsgi_harakiri|default(5)}}
+harakiri = {{app_harakiri|default(5)}}
 buffer-size=32768
-{% if uwsgi_enable_threads %}
+{% if app_enable_threads|default(false) %}
 enable-threads = 
 {% endif %}
-{% if uwsgi_options is defined %}
-{% for option in uwsgi_options %}
+{% if app_uwsgi_options is defined %}
+{% for option in app_uwsgi_options %}
 {{option}}{% if "=" not in option %} ={% endif %}
 {% endfor %}
 {% endif %}
 
-{% for mule in range(uwsgi_mules) %}
+{% for mule in range(app_mules|default(0)) %}
 mule = 
 {% endfor %}
 #umask = 227

uwsgi-python/vars/protokollsystem.yml

+app_name: protokollsystem
+app_user: protokolle
+app_group: protokolle
+app_home: /var/www/protokollsystem
+app_path: /var/www/protokollsystem/program
+app_python_version: 3
+app_mules: 1
+app_enable_threads: true
+app_venv: /var/www/protokollsystem/program/
+app_program: server.py
+app_callable: app
+app_command: ""
+app_mountpoint: /
+
+app_db_name: protokollsystem
+app_db_type: postgres
+
+app_additional_software:
+  - "libxml2-dev"
+  - "libxslt-dev"
+  - "fontconfig"
+  - "tex-gyre"
+
+app_deploy_key: "{{ inventory_dir }}/files/deploy-keys/protokollsystem"
+app_git_url: "git@git.fsmpi.rwth-aachen.de:protokollsystem/proto3.git"
+app_git_version: HEAD
+
+app_config_file: config.py
+app_secret_config: true
+app_secret_config_keys: ["security_key"]
+
+protokolle_celery_broker: 'redis://localhost:6379/0'
+protokolle_celery_concurrency: 4
+
+protokolle_sentry_dsn: "https://3bbe6efca11645488bc1fb7297013e3a:4747c40e928d401e8e5e9aac49d3b42d@sentry.fsmpi.rwth-aachen.de//4"