From 4ed24ebee81bbcf6834772d5af0417f074766b54 Mon Sep 17 00:00:00 2001
From: Hinrikus Wolf <hinrikus@fsmpi.rwth-aachen.de>
Date: Wed, 6 Mar 2019 23:45:09 +0100
Subject: [PATCH] clean up and add documentation
---
README.ganeti.md | 67 +++++++++++++++++++
README.md | 73 +++++----------------
packages.yml | 23 -------
requirements.txt | 1 -
scripts/build.py | 166 -----------------------------------------------
5 files changed, 84 insertions(+), 246 deletions(-)
create mode 100644 README.ganeti.md
delete mode 100644 packages.yml
delete mode 100644 requirements.txt
delete mode 100755 scripts/build.py
diff --git a/README.ganeti.md b/README.ganeti.md
new file mode 100644
index 0000000..ce80f88
--- /dev/null
+++ b/README.ganeti.md
@@ -0,0 +1,67 @@
+# Patches for Ganeti
+
+This repository contains our patches for ganeti which we apply on top of the already patched debian package.
+We retrieve the package source from debian, add some patches, build, and copy to our debian repository.
+After installation from our repository, we need to pin those to prevent updating from debian's regular sources.
+See apt's documentation on pinning.
+
+
+## Interesting Links
+
+Rebuilding an existing package is described here:
+https://wiki.debian.org/BuildingTutorial
+
+More information on building:
+https://wiki.debian.org/HowToPackageForDebian
+
+The ganeti package uses `quilt` to manage its patches. See this tutorial:
+https://raphaelhertzog.com/2012/08/08/how-to-use-quilt-to-manage-patches-in-debian-packages/
+
+
+## Rebuilding Ganeti
+
+It boils down to the following steps:
+
+- `apt-get source ganeti`
+- `cd ganeti-2.15.2/debian`
+- `quilt import ../remove_blockdev_from_movable_list.patch` and others
+- `dch -n` creates a new changelog entry
+- `debuild -b -uc -us` builds the package
+
+At least according to my zsh history.
+
+If you are building from an already changed version, but without that particular changelog, use:
+
+- `dch --auto-nmr --newversion $newversion`
+
+Ideally, you copy the old changelog to the new one. Look up the old version number!
+Add "Non-maintainer upload." manually to your list of changes.
+
+
+## Patch Details
+
+### `fix_ceph_showmap.patch`
+
+Fixes an incompatibility with current ceph versions.
+
+See also: https://github.com/ganeti/ganeti/issues/1233
+
+### `increase_max_disk_count.patch`
+
+*sigh*
+
+There is a constant number of allowed disks that may be attached to a VM. We increase this limit.
+
+### `remove_blockdev_from_mirrored_list.patch` and `remove_blockdev_from_movable_list.patch`
+
+Removes block devices from lists of mirrored and movable devices. This prevents such VMs from being migrated.
+
+This may not solve the general case, but is makes it compatible with our usage.
+
+
+### `add_rbd_cache_support.patch`
+
+Add caching support for rbd devices. This was disabled for arbritrary external storage.
+But this does not make sense for rbd.
+
+
diff --git a/README.md b/README.md
index ce80f88..571729d 100644
--- a/README.md
+++ b/README.md
@@ -1,67 +1,28 @@
-# Patches for Ganeti
+# CI-Building of Debian packages
-This repository contains our patches for ganeti which we apply on top of the already patched debian package.
-We retrieve the package source from debian, add some patches, build, and copy to our debian repository.
-After installation from our repository, we need to pin those to prevent updating from debian's regular sources.
-See apt's documentation on pinning.
+[](https://git.fsmpi.rwth-aachen.de/infra/packages/commits/master)
+This repository contains build scripts and patches for our self-build packages.
+The CI config runs these scripts which download, apply patches and build the packages.
-## Interesting Links
+## Build Scripts
-Rebuilding an existing package is described here:
-https://wiki.debian.org/BuildingTutorial
+The build scripts are in the `scripts/` directory. They shall be named as the packages.
+These scripts will be run in the folder where the CI has checked out the repository. Ensure that the script only uses relative paths.
-More information on building:
-https://wiki.debian.org/HowToPackageForDebian
+## Patches
-The ganeti package uses `quilt` to manage its patches. See this tutorial:
-https://raphaelhertzog.com/2012/08/08/how-to-use-quilt-to-manage-patches-in-debian-packages/
+The patches are in the `patches/` directory. For each package there shall be a subfolder with the name of the package if needed.
+## Build
-## Rebuilding Ganeti
+The packages are uploaded to the CI artifacts after build.
-It boils down to the following steps:
-
-- `apt-get source ganeti`
-- `cd ganeti-2.15.2/debian`
-- `quilt import ../remove_blockdev_from_movable_list.patch` and others
-- `dch -n` creates a new changelog entry
-- `debuild -b -uc -us` builds the package
-
-At least according to my zsh history.
-
-If you are building from an already changed version, but without that particular changelog, use:
-
-- `dch --auto-nmr --newversion $newversion`
-
-Ideally, you copy the old changelog to the new one. Look up the old version number!
-Add "Non-maintainer upload." manually to your list of changes.
-
-
-## Patch Details
-
-### `fix_ceph_showmap.patch`
-
-Fixes an incompatibility with current ceph versions.
-
-See also: https://github.com/ganeti/ganeti/issues/1233
-
-### `increase_max_disk_count.patch`
-
-*sigh*
-
-There is a constant number of allowed disks that may be attached to a VM. We increase this limit.
-
-### `remove_blockdev_from_mirrored_list.patch` and `remove_blockdev_from_movable_list.patch`
-
-Removes block devices from lists of mirrored and movable devices. This prevents such VMs from being migrated.
-
-This may not solve the general case, but is makes it compatible with our usage.
-
-
-### `add_rbd_cache_support.patch`
-
-Add caching support for rbd devices. This was disabled for arbritrary external storage.
-But this does not make sense for rbd.
+## Packages
+- `adcli` uses upstream patches from master (GPL licensed)
+- [`ganeti`](README.ganeti.md)
+- `grml-zsh-config` takes the `grml-etc-core` package and throws out everything but zsh config
+- `nginx` builds nginx with the nginx-shib-module
+- `sssd` backports the sssd-package from buster
diff --git a/packages.yml b/packages.yml
deleted file mode 100644
index 163cf33..0000000
--- a/packages.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-maintainer:
- name: FSMPI Admin-Team
- mail: admin@fsmpi.rwth-aachen.de
-upload_target: repo@web.fsmpi.rwth-aachen.de:/srv/repo/incoming/
-packages:
- - name: adcli
- version: 0.8.2-1.1-fsmpi
- changelog: Apply the unreleased upstream-patches, fixing RT#100
- additional_content:
- - patch_dir: patches/adcli
- target_dir: debian
- - name: nginx
- version: 1.10.3-1+deb9u2.2-fsmpi
- changelog: Add libnginx-mod-http-shibboleth
- additional_content:
- - git_url: "https://github.com/nginx-shib/nginx-http-shibboleth.git"
- target_dir: debian/modules
- - copy:
- src: libnginx-mod.nginx.skeleton
- dest: libnginx-mod-http-shibboleth.nginx
- target_dir: debian
- - target_dir: .
- patch: patches/nginx/00-add-module.patch
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 4a28555..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-PyYAML==3.13
diff --git a/scripts/build.py b/scripts/build.py
deleted file mode 100755
index e0a95a5..0000000
--- a/scripts/build.py
+++ /dev/null
@@ -1,166 +0,0 @@
-#!/usr/bin/env python3
-
-import locale
-locale.setlocale(locale.LC_ALL, "en_US.UTF-8")
-
-import os
-import subprocess as sp
-import re
-import shutil
-import tempfile
-import yaml
-
-
-def load_config():
- with open("packages.yml", "r") as config_file:
- return yaml.load(config_file)
-
-
-PKG_NAME_PATTERN = r"[a-zA-Z0-9.+-]+"
-EPOCH_PATTERN = r"[0-9]+:"
-UPSTREAM_VERSION_PATTERN = r"[a-zA-Z0-9.+-:]+"
-DEBIAN_VERSION_PATTERN = r"[a-zA-Z0-9.+~]+"
-EXTRACT_PATTERN = (r"extracting (?P<name>{pkg_name}) in (?P<dir>{pkg_name}-{version})"
- .format(pkg_name=PKG_NAME_PATTERN, version=UPSTREAM_VERSION_PATTERN))
-
-
-def run_checked(command, **kwargs):
- print(os.getcwd(), command)
- try:
- return sp.run(
- command, check=True,
- stdout=sp.PIPE, stderr=sp.PIPE, universal_newlines=True,
- **kwargs)
- except sp.CalledProcessError as error:
- print(error.stdout)
- print(error.stderr)
- raise
-
-
-def apply_patches(abs_patch_dir):
- for patch in sorted(os.listdir(abs_patch_dir)):
- patch_file = os.path.join(abs_patch_dir, patch)
- run_checked(["quilt", "import", patch_file])
-
-
-def build_package(repo_dir, package_dir, name,
- patch_dir=None, version=None, changelog=None, additional_content=None):
- with tempfile.TemporaryDirectory(dir=os.path.abspath(".")) as tempdir:
- os.chdir(tempdir)
- result = run_checked(["apt-get", "source", name])
- source_dir_candidates = [
- line for line in result.stdout.splitlines() if "extracting" in line]
- if len(source_dir_candidates) != 1:
- raise ValueError("Got inconclusive candidate directories: {}".format(
- source_dir_candidates))
- source_dir_match = re.search(EXTRACT_PATTERN, source_dir_candidates[0])
- if source_dir_match is None:
- raise ValueError("Cannot get extraction directory from {}".format(
- source_dir_candidates[0]))
- pkg_dir = source_dir_match.group("dir")
- pkg_name = source_dir_match.group("name")
- if pkg_name != name:
- raise ValueError(
- "Extracted package {} is not expected package {}".format(
- pkg_name, name))
-
- print("Extracted name", pkg_dir, pkg_name)
-
- workdir = os.path.join(tempdir, pkg_dir)
-
- if additional_content is not None:
- for content in additional_content:
- target_dir = content["target_dir"]
- os.chdir(os.path.join(workdir, target_dir))
- if "patch_dir" in content:
- abs_patch_dir = os.path.join(
- repo_dir, content["patch_dir"])
- apply_patches(abs_patch_dir)
- elif "patch" in content:
- abs_patch_dir = os.path.join(repo_dir, content["patch"])
- with open(abs_patch_dir, "r") as patch_file:
- run_checked(["patch", "-d", ".", "-p0"],
- input=patch_file.read())
- elif "git_url" in content:
- run_checked(["git", "clone", content["git_url"]])
- elif "copy" in content:
- copy = content["copy"]
- shutil.copy(copy["src"], copy["dest"])
- else:
- raise ValueError(
- "Unknown kind of additional content: {}".format(
- content))
-
- os.chdir(workdir)
-
- command = ["debchange", "--preserve"]
- if version is None:
- command.append("--nmu")
- else:
- command.extend(["--newversion", version])
- def _get_log_entries():
- if changelog:
- yield changelog
- yield "Non-maintainer upload"
- for entry in _get_log_entries():
- run_checked(command + [entry])
-
- command = ["apt-get", "build-dep", "-y", name]
- run_checked(command)
-
- command = ["debuild", "-b", "-uc", "-us"]
- run_checked(command)
-
- deb_packages = [
- filename
- for filename in os.listdir(tempdir)
- if filename.endswith(".deb")
- ]
- for filename in deb_packages:
- shutil.move(os.path.join(tempdir, filename), package_dir)
-
-
-def upload_packages(package_dir, upload_target, key_file=None):
- ssh_args = ["ssh", "-o", "StrictHostKeyChecking=no"]
- if key_file:
- ssh_args.extend(["-i", key_file])
- if not package_dir.endswith("/"):
- package_dir += "/"
- command = [
- "rsync",
- "--recursive",
- "-e", " ".join(ssh_args),
- package_dir,
- upload_target,
- ]
- run_checked(command)
-
-
-def main():
- repo_dir = os.getcwd()
- config = load_config()
- maintainer = config.get("maintainer", None)
- if maintainer is not None:
- name = maintainer.get("name", None)
- mail = maintainer.get("mail", None)
- if name:
- os.environ["DEBFULLNAME"] = name
- if mail:
- os.environ["DEBEMAIL"] = mail
-
- package_dir = os.path.join(repo_dir, "packages")
- os.makedirs(package_dir, exist_ok=True)
-
- for package in config["packages"]:
- build_package(repo_dir=repo_dir, package_dir=package_dir, **package)
- os.chdir(repo_dir)
-
- import pprint
- print("Built packages:")
- pprint.pprint(sorted(os.listdir(package_dir)))
-
- upload_packages(package_dir, config["upload_target"], key_file="~/.ssh/id_ed25519")
-
-if __name__ == "__main__":
- main()
-
--
GitLab