From 4360ad5ca703c99723c72d16e0da3f4ca92f92d9 Mon Sep 17 00:00:00 2001 From: Sean Whitton Date: Tue, 8 Jun 2021 14:59:51 -0700 Subject: add properties to build and configure sbuild schroots Based on properties I wrote for Propellor. Signed-off-by: Sean Whitton --- consfigurator.asd | 3 + src/package.lisp | 34 ++++++++ src/property/ccache.lisp | 61 +++++++++++++++ src/property/sbuild.lisp | 193 ++++++++++++++++++++++++++++++++++++++++++++++ src/property/schroot.lisp | 65 ++++++++++++++++ 5 files changed, 356 insertions(+) create mode 100644 src/property/ccache.lisp create mode 100644 src/property/sbuild.lisp create mode 100644 src/property/schroot.lisp diff --git a/consfigurator.asd b/consfigurator.asd index 8baa2c0..5dc737b 100644 --- a/consfigurator.asd +++ b/consfigurator.asd @@ -53,6 +53,9 @@ (:file "src/property/hostname") (:file "src/property/network") (:file "src/property/libvirt") + (:file "src/property/ccache") + (:file "src/property/schroot") + (:file "src/property/sbuild") (:file "src/connection/shell-wrap") (:file "src/connection/fork") (:file "src/connection/rehome") diff --git a/src/package.lisp b/src/package.lisp index 64eba15..563c752 100644 --- a/src/package.lisp +++ b/src/package.lisp @@ -611,6 +611,40 @@ #:kvm-boots-chroot #:kvm-boots-chroot.)) +(defpackage :consfigurator.property.ccache + (:use #:cl #:consfigurator) + (:local-nicknames (#:os #:consfigurator.property.os) + (#:file #:consfigurator.property.file) + (#:apt #:consfigurator.property.apt)) + (:export #:installed + #:has-limits + #:group-cache)) + +(defpackage :consfigurator.property.schroot + (:use #:cl #:consfigurator) + (:local-nicknames (#:os #:consfigurator.property.os) + (#:file #:consfigurator.property.file) + (#:apt #:consfigurator.property.apt)) + (:export #:installed + #:uses-overlays + #:overlays-in-tmpfs)) + +(defpackage :consfigurator.property.sbuild + (:use #:cl #:alexandria #:consfigurator) + (:local-nicknames (#:os #:consfigurator.property.os) + (#:file #:consfigurator.property.file) + (#:chroot #:consfigurator.property.chroot) + (#:user #:consfigurator.property.user) + (#:apt #:consfigurator.property.apt) + (#:ccache #:consfigurator.property.ccache) + (#:schroot #:consfigurator.property.schroot) + (#:periodic #:consfigurator.property.periodic)) + (:export #:installed + #:usable-by + #:built + #:built. + #:standard-debian-schroot)) + (defpackage :consfigurator.connection.local (:use #:cl #:consfigurator #:alexandria) (:export #:local-connection)) diff --git a/src/property/ccache.lisp b/src/property/ccache.lisp new file mode 100644 index 0000000..9922df1 --- /dev/null +++ b/src/property/ccache.lisp @@ -0,0 +1,61 @@ +;;; Consfigurator -- Lisp declarative configuration management system + +;;; Copyright (C) 2021 Sean Whitton + +;;; This file is free software; you can redistribute it and/or modify +;;; it under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3, or (at your option) +;;; any later version. + +;;; This file is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. + +;;; You should have received a copy of the GNU General Public License +;;; along with this program. If not, see . + +(in-package :consfigurator.property.ccache) +(named-readtables:in-readtable :consfigurator) + +(defproplist installed :posix () + (:desc "ccache installed") + (os:etypecase + (debianlike (apt:installed "ccache")))) + +(defprop has-limits :posix (cache + &key (max-size nil max-size-supplied-p) + (max-files nil max-files-supplied-p)) + "Set limits on a given ccache. +See ccache(1) for the format of MAX-SIZE." + (:desc (format nil "~A has max size ~D & max files ~D" + cache max-size max-files)) + (:apply + (with-change-if-changes-file-content + ((merge-pathnames "ccache.conf" (ensure-directory-pathname cache))) + ;; Let ccache(1) handle editing and deduplicating the config file, etc. + (mrun "ccache" :env `(:CCACHE_DIR ,cache) + (and max-size-supplied-p + (strcat "--max-size=" (or max-size "0"))) + (and max-files-supplied-p + (strcat "--max-files=" (or max-files "0"))))))) + +(defpropspec group-cache :posix + (group &key (max-size nil max-size-supplied-p) + (max-files nil max-files-supplied-p) + &aux (dir (ensure-directory-pathname + (strcat "/var/cache/ccache-" group)))) + "Configures a ccache in /var/cache for a group." + (:desc #?"ccache for group ${group} exists") + `(with-unapply + (installed) + (file:directory-exists ,dir) + (file:has-mode ,dir #o2775) + (file:has-ownership ,dir :user "root" :group ,group) + ,@(and (or max-size-supplied-p max-files-supplied-p) + `((has-limits ,dir + ,@(and max-size-supplied-p + `(:max-size ,max-size)) + ,@(and max-files-supplied-p + `(:max-files ,max-files))))) + :unapply (file:directory-does-not-exist ,dir))) diff --git a/src/property/sbuild.lisp b/src/property/sbuild.lisp new file mode 100644 index 0000000..17ec67e --- /dev/null +++ b/src/property/sbuild.lisp @@ -0,0 +1,193 @@ +;;; Consfigurator -- Lisp declarative configuration management system + +;;; Copyright (C) 2016, 2021 Sean Whitton + +;;; This file is free software; you can redistribute it and/or modify +;;; it under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3, or (at your option) +;;; any later version. + +;;; This file is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. + +;;; You should have received a copy of the GNU General Public License +;;; along with this program. If not, see . + +(in-package :consfigurator.property.sbuild) +(named-readtables:in-readtable :consfigurator) + +(defproplist installed :posix () + "Ensure that sbuild and associated utilities are installed." + (:desc "sbuild and associated utilities installed") + (os:etypecase + (debianlike (apt:installed "piuparts" "autopkgtest" "lintian" "sbuild")))) + +(defproplist usable-by :posix (username) + "Add a user to the sbuild group in order to use sbuild." + (:desc #?"sbuild usable by ${username}") + (installed) + (user:has-groups username "sbuild")) + +(defproplist %sbuild-ccache-has-some-limits :posix () + "Set a default limit on the sbuild ccache, only if the ccache does not already +exist, so that the user can easily override this default." + (:desc #?"Default limits on sbuild ccache") + (:check (remote-exists-p "/var/cache/ccache-sbuild")) + (ccache:has-limits "/var/cache/ccache-sbuild" :max-size "2Gi")) + +(defpropspec built :lisp + (options properties + &aux (host + (make-host + :propspec + (append-propspecs + properties + (make-propspec + :systems nil + :propspec + '(desc "Build packages installed into chroot" + (os:etypecase + (debianlike (apt:installed "eatmydata" "ccache")))))))) + (os (get-hostattrs-car :os host)) + (suite (os:debian-suite os)) + (arch (os:debian-architecture os))) + "Build and configure a schroot for use with sbuild. +For convenience we set up several enhancements, such as ccache and eatmydata. +In the case of Debian, we assume you are building for Debian stretch or newer, +and we assume that you have sbuild 0.71.0 or later and, if overlays are +enabled, Linux 3.18 or newer. + +OPTIONS is a plist of keyword parameters: + + - :USE-CCACHE -- whether builds using the schroot should use ccache. ccache + is generally useful but breaks building some packages; this option allows + you to toggle it on and off for particular schroots. Defaults to t. + + - :CHROOT-OPTIONS -- passed on to CHROOT:OS-BOOTSTRAPPED-FOR, which see. + +PROPERTIES should specify, at a minimum, the operating system for the schroot. + +Example usage: + + (os:debian-stable \"bullseye\" :amd64) + (apt:uses-local-cacher) + (apt:mirror \"...\") + (sbuild:usable-by \"spwhitton\") + (schroot:overlays-in-tmpfs) + (sbuild:built. nil + (os:debian-unstable :amd64) + (sbuild:standard-debian-schroot) + (apt:uses-parent-proxy) + (apt:uses-parent-mirror)) + +To take advantage of the piuparts and autopkgtest support, add to your +~/.sbuildrc: + + $piuparts_opts = [ + '--no-eatmydata', + '--schroot', + '%r-%a-sbuild', + '--log-level=info', + ]; + + $autopkgtest_root_args = \"\"; + $autopkgtest_opts = [\"--\", \"schroot\", \"%r-%a-sbuild\"];" + (:desc (format nil "Built sbuild schroot for ~A/~A" suite arch)) + (destructuring-bind + (&key (use-ccache t) chroot-options + &aux + (chroot + (ensure-pathname + (format nil "~A-~A" suite arch) + :ensure-directory t :ensure-absolute t :defaults #P"/srv/chroot/")) + (desc (format nil "~A/~A autobuilder" suite arch)) + (conf (ensure-pathname + (format nil "~A-~A-sbuild-consfigurator" suite arch) + :ensure-absolute t :defaults #P"/etc/schroot/chroot.d/"))) + options + `(with-unapply + (installed) + + ;; ccache + ,@(and use-ccache '((%sbuild-ccache-has-some-limits) + (ccache:group-cache "sbuild"))) + (desc + "ccache mounted in sbuild schroots" + (file:contains-lines "/etc/schroot/sbuild/fstab" + "/var/cache/ccache-sbuild /var/cache/ccache-sbuild none rw,bind 0 0")) + ;; Script from . + (file:has-content "/var/cache/ccache-sbuild/sbuild-setup" +#>EOF>#!/bin/sh + +export CCACHE_DIR=/var/cache/ccache-sbuild +export CCACHE_UMASK=002 +export CCACHE_COMPRESS=1 +unset CCACHE_HARDLINK +export PATH="/usr/lib/ccache:$PATH" + +exec "$@" +EOF :mode #o755) + + ;; schroot + (chroot:os-bootstrapped-for ,chroot-options ,chroot ,host) + (desc + ,(strcat "schroot configuration for " desc) + (file:contains-ini-settings + ,conf + ,@(mapcar + (curry #'cons (format nil "~A-~A-sbuild" suite arch)) + `(("description" ,desc) + ("groups" "root,sbuild") + ("root-groups" "root,sbuild") + ("profile" "sbuild") + ("type" "directory") + ("directory" ,(drop-trailing-slash (unix-namestring chroot))) + ,@(and (get-hostattrs-car 'schroot:uses-overlays) + `(("union-type" "overlay"))) + + ;; If we're building a sid chroot, add useful aliases. In order + ;; to avoid more than one schroot getting the same aliases, we + ;; only do this if the arch of the chroot equals the host arch. + ,@(and (string= suite "unstable") + (string= arch (os:debian-architecture + (get-hostattrs-car :os))) + `(("aliases" + ,(format + nil "~@{~A~^,~}" + "sid" + ;; If the user wants to build for experimental, they + ;; would use their sid chroot and sbuild's + ;; --extra-repository option to enable experimental. + "rc-buggy" + "experimental" + ;; We assume that building for UNRELEASED means + ;; building for unstable. + "UNRELEASED" + ;; The following is for dgit compatibility. + (strcat "UNRELEASED-" + (os:debian-architecture os) + "-sbuild"))))) + + ("command-prefix" + ,(if use-ccache + "/var/cache/ccache-sbuild/sbuild-setup:eatmydata" + "eatmydata")))))) + + ;; TODO We should kill any sessions still using the chroot before + ;; destroying it (as suggested by sbuild-destroychroot(8)). + :unapply + (unapply (chroot:os-bootstrapped-for ,chroot-options ,chroot ,host)) + (file:does-not-exist ,conf)))) + +;; Here we combine Propellor's Sbuild.osDebianStandard and Sbuild.update. +(defpropspec standard-debian-schroot :posix (&optional (period :weekly)) + "Properties that will be wanted in almost any Debian sbuild schroot, but not +in sbuild schroots for other operating systems. + +Includes replacing use of sbuild-update(1)." + (:desc "Standard Debian sbuild properties") + `(eseqprops (apt:standard-sources.list) + (periodic:at-most ,period "sbuild schroot updated" + (apt:updated) (apt:upgraded) (apt:autoremoved)))) diff --git a/src/property/schroot.lisp b/src/property/schroot.lisp new file mode 100644 index 0000000..2e6cb40 --- /dev/null +++ b/src/property/schroot.lisp @@ -0,0 +1,65 @@ +;;; Consfigurator -- Lisp declarative configuration management system + +;;; Copyright (C) 2016, 2021 Sean Whitton + +;;; This file is free software; you can redistribute it and/or modify +;;; it under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3, or (at your option) +;;; any later version. + +;;; This file is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. + +;;; You should have received a copy of the GNU General Public License +;;; along with this program. If not, see . + +(in-package :consfigurator.property.schroot) +(named-readtables:in-readtable :consfigurator) + +(defproplist installed :posix () + "Ensure that schroot(1) is installed." + (:desc "schroot(1) installed") + (os:etypecase + (debianlike (apt:installed "schroot")))) + +(defprop uses-overlays :posix () + "Indicate that schroots on a host should use 'union-type=overlay'. + +Adding this property does not actually ensure that the line +'union-type=overlay' is present in any schroot config files. See SBUILD:BUILT +for example usage, via SCHROOT:OVERLAYS-IN-TMPFS." + (:desc "schroots on host use union-type=overlay") + (:hostattrs (push-hostattrs 'uses-overlays t))) + +(defprop overlays-in-tmpfs :posix () + "Configure schroot(1) such that all schroots with 'union-type=overlay' in +their configuration will run their overlays in a tmpfs. Unapplicable, so if +the package you are working on FTBFS when overlays are in tmpfs, you can +toggle this off for a host, and then toggle it back on again later. + +Implicitly sets SCHROOT:USES-OVERLAYS. + +Shell script from ." + (:desc "schroot overlays in tmpfs") + (:hostattrs (push-hostattrs 'uses-overlays t)) + (:apply (file:has-content "/etc/schroot/setup.d/04tmpfs" +#>EOF>#!/bin/sh + +set -e + +. "$SETUP_DATA_DIR/common-data" +. "$SETUP_DATA_DIR/common-functions" +. "$SETUP_DATA_DIR/common-config" + + +if [ $STAGE = "setup-start" ]; then + mount -t tmpfs overlay /var/lib/schroot/union/overlay +elif [ $STAGE = "setup-recover" ]; then + mount -t tmpfs overlay /var/lib/schroot/union/overlay +elif [ $STAGE = "setup-stop" ]; then + umount -f /var/lib/schroot/union/overlay +fi +EOF :mode #o755)) + (:unapply (file:does-not-exist "/etc/schroot/setup.d/04tmpfs"))) -- cgit v1.2.3