add properties to build and configure sbuild schroots

Based on properties I wrote for Propellor.

Signed-off-by: Sean Whitton <spwhitton@spwhitton.name>

5 files changed, 356 insertions, 0 deletions

diff --git a/consfigurator.asd b/consfigurator.asd
index 8baa2c0..5dc737b 100644
--- a/consfigurator.asd
+++ b/consfigurator.asd
@@ -53,6 +53,9 @@
                (:file "src/property/hostname")
                (:file "src/property/network")
                (:file "src/property/libvirt")
+               (:file "src/property/ccache")
+               (:file "src/property/schroot")
+               (:file "src/property/sbuild")
                (:file "src/connection/shell-wrap")
 	       (:file "src/connection/fork")
                (:file "src/connection/rehome")
diff --git a/src/package.lisp b/src/package.lisp
index 64eba15..563c752 100644
--- a/src/package.lisp
+++ b/src/package.lisp
@@ -611,6 +611,40 @@
            #:kvm-boots-chroot
            #:kvm-boots-chroot.))
 
+(defpackage :consfigurator.property.ccache
+  (:use #:cl #:consfigurator)
+  (:local-nicknames (#:os        #:consfigurator.property.os)
+                    (#:file      #:consfigurator.property.file)
+                    (#:apt       #:consfigurator.property.apt))
+  (:export #:installed
+           #:has-limits
+           #:group-cache))
+
+(defpackage :consfigurator.property.schroot
+  (:use #:cl #:consfigurator)
+  (:local-nicknames (#:os        #:consfigurator.property.os)
+                    (#:file      #:consfigurator.property.file)
+                    (#:apt       #:consfigurator.property.apt))
+  (:export #:installed
+           #:uses-overlays
+           #:overlays-in-tmpfs))
+
+(defpackage :consfigurator.property.sbuild
+  (:use #:cl #:alexandria #:consfigurator)
+  (:local-nicknames (#:os        #:consfigurator.property.os)
+                    (#:file      #:consfigurator.property.file)
+                    (#:chroot    #:consfigurator.property.chroot)
+                    (#:user      #:consfigurator.property.user)
+                    (#:apt       #:consfigurator.property.apt)
+                    (#:ccache    #:consfigurator.property.ccache)
+                    (#:schroot   #:consfigurator.property.schroot)
+                    (#:periodic  #:consfigurator.property.periodic))
+  (:export #:installed
+           #:usable-by
+           #:built
+           #:built.
+           #:standard-debian-schroot))
+
 (defpackage :consfigurator.connection.local
   (:use #:cl #:consfigurator #:alexandria)
   (:export #:local-connection))
diff --git a/src/property/ccache.lisp b/src/property/ccache.lisp
new file mode 100644
index 0000000..9922df1
--- /dev/null
+++ b/src/property/ccache.lisp
@@ -0,0 +1,61 @@
+;;; Consfigurator -- Lisp declarative configuration management system
+
+;;; Copyright (C) 2021  Sean Whitton <spwhitton@spwhitton.name>
+
+;;; This file is free software; you can redistribute it and/or modify
+;;; it under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3, or (at your option)
+;;; any later version.
+
+;;; This file is distributed in the hope that it will be useful,
+;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+
+;;; You should have received a copy of the GNU General Public License
+;;; along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+(in-package :consfigurator.property.ccache)
+(named-readtables:in-readtable :consfigurator)
+
+(defproplist installed :posix ()
+  (:desc "ccache installed")
+  (os:etypecase
+    (debianlike (apt:installed "ccache"))))
+
+(defprop has-limits :posix (cache
+                            &key (max-size nil max-size-supplied-p)
+                            (max-files nil max-files-supplied-p))
+  "Set limits on a given ccache.
+See ccache(1) for the format of MAX-SIZE."
+  (:desc (format nil "~A has max size ~D & max files ~D"
+                 cache max-size max-files))
+  (:apply
+   (with-change-if-changes-file-content
+       ((merge-pathnames "ccache.conf" (ensure-directory-pathname cache)))
+     ;; Let ccache(1) handle editing and deduplicating the config file, etc.
+     (mrun "ccache" :env `(:CCACHE_DIR ,cache)
+           (and max-size-supplied-p
+                (strcat "--max-size=" (or max-size "0")))
+           (and max-files-supplied-p
+                (strcat "--max-files=" (or max-files "0")))))))
+
+(defpropspec group-cache :posix
+    (group &key (max-size nil max-size-supplied-p)
+           (max-files nil max-files-supplied-p)
+           &aux (dir (ensure-directory-pathname
+                      (strcat "/var/cache/ccache-" group))))
+  "Configures a ccache in /var/cache for a group."
+  (:desc #?"ccache for group ${group} exists")
+  `(with-unapply
+     (installed)
+     (file:directory-exists ,dir)
+     (file:has-mode ,dir #o2775)
+     (file:has-ownership ,dir :user "root" :group ,group)
+     ,@(and (or max-size-supplied-p max-files-supplied-p)
+            `((has-limits ,dir
+                          ,@(and max-size-supplied-p
+                                 `(:max-size ,max-size))
+                          ,@(and max-files-supplied-p
+                                 `(:max-files ,max-files)))))
+     :unapply (file:directory-does-not-exist ,dir)))
diff --git a/src/property/sbuild.lisp b/src/property/sbuild.lisp
new file mode 100644
index 0000000..17ec67e
--- /dev/null
+++ b/src/property/sbuild.lisp
@@ -0,0 +1,193 @@
+;;; Consfigurator -- Lisp declarative configuration management system
+
+;;; Copyright (C) 2016, 2021  Sean Whitton <spwhitton@spwhitton.name>
+
+;;; This file is free software; you can redistribute it and/or modify
+;;; it under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3, or (at your option)
+;;; any later version.
+
+;;; This file is distributed in the hope that it will be useful,
+;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+
+;;; You should have received a copy of the GNU General Public License
+;;; along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+(in-package :consfigurator.property.sbuild)
+(named-readtables:in-readtable :consfigurator)
+
+(defproplist installed :posix ()
+  "Ensure that sbuild and associated utilities are installed."
+  (:desc "sbuild and associated utilities installed")
+  (os:etypecase
+    (debianlike (apt:installed "piuparts" "autopkgtest" "lintian" "sbuild"))))
+
+(defproplist usable-by :posix (username)
+  "Add a user to the sbuild group in order to use sbuild."
+  (:desc #?"sbuild usable by ${username}")
+  (installed)
+  (user:has-groups username "sbuild"))
+
+(defproplist %sbuild-ccache-has-some-limits :posix ()
+  "Set a default limit on the sbuild ccache, only if the ccache does not already
+exist, so that the user can easily override this default."
+  (:desc #?"Default limits on sbuild ccache")
+  (:check (remote-exists-p "/var/cache/ccache-sbuild"))
+  (ccache:has-limits "/var/cache/ccache-sbuild" :max-size "2Gi"))
+
+(defpropspec built :lisp
+    (options properties
+             &aux (host
+                   (make-host
+                    :propspec
+                    (append-propspecs
+                     properties
+                     (make-propspec
+                      :systems nil
+                      :propspec
+                      '(desc "Build packages installed into chroot"
+                        (os:etypecase
+                          (debianlike (apt:installed "eatmydata" "ccache"))))))))
+             (os (get-hostattrs-car :os host))
+             (suite (os:debian-suite os))
+             (arch (os:debian-architecture os)))
+  "Build and configure a schroot for use with sbuild.
+For convenience we set up several enhancements, such as ccache and eatmydata.
+In the case of Debian, we assume you are building for Debian stretch or newer,
+and we assume that you have sbuild 0.71.0 or later and, if overlays are
+enabled, Linux 3.18 or newer.
+
+OPTIONS is a plist of keyword parameters:
+
+  - :USE-CCACHE -- whether builds using the schroot should use ccache.  ccache
+    is generally useful but breaks building some packages; this option allows
+    you to toggle it on and off for particular schroots.  Defaults to t.
+
+  - :CHROOT-OPTIONS -- passed on to CHROOT:OS-BOOTSTRAPPED-FOR, which see.
+
+PROPERTIES should specify, at a minimum, the operating system for the schroot.
+
+Example usage:
+
+    (os:debian-stable \"bullseye\" :amd64)
+    (apt:uses-local-cacher)
+    (apt:mirror \"...\")
+    (sbuild:usable-by \"spwhitton\")
+    (schroot:overlays-in-tmpfs)
+    (sbuild:built. nil
+      (os:debian-unstable :amd64)
+      (sbuild:standard-debian-schroot)
+      (apt:uses-parent-proxy)
+      (apt:uses-parent-mirror))
+
+To take advantage of the piuparts and autopkgtest support, add to your
+~/.sbuildrc:
+
+    $piuparts_opts = [
+        '--no-eatmydata',
+        '--schroot',
+        '%r-%a-sbuild',
+        '--log-level=info',
+        ];
+
+    $autopkgtest_root_args = \"\";
+    $autopkgtest_opts = [\"--\", \"schroot\", \"%r-%a-sbuild\"];"
+  (:desc (format nil "Built sbuild schroot for ~A/~A" suite arch))
+  (destructuring-bind
+      (&key (use-ccache t) chroot-options
+       &aux
+         (chroot
+          (ensure-pathname
+           (format nil "~A-~A" suite arch)
+           :ensure-directory t :ensure-absolute t :defaults #P"/srv/chroot/"))
+         (desc (format nil "~A/~A autobuilder" suite arch))
+         (conf (ensure-pathname
+                (format nil "~A-~A-sbuild-consfigurator" suite arch)
+                :ensure-absolute t :defaults #P"/etc/schroot/chroot.d/")))
+      options
+    `(with-unapply
+         (installed)
+
+       ;; ccache
+       ,@(and use-ccache '((%sbuild-ccache-has-some-limits)
+                           (ccache:group-cache "sbuild")))
+       (desc
+        "ccache mounted in sbuild schroots"
+        (file:contains-lines "/etc/schroot/sbuild/fstab"
+          "/var/cache/ccache-sbuild /var/cache/ccache-sbuild none rw,bind 0 0"))
+       ;; Script from <https://wiki.debian.org/sbuild>.
+       (file:has-content "/var/cache/ccache-sbuild/sbuild-setup"
+#>EOF>#!/bin/sh
+
+export CCACHE_DIR=/var/cache/ccache-sbuild
+export CCACHE_UMASK=002
+export CCACHE_COMPRESS=1
+unset CCACHE_HARDLINK
+export PATH="/usr/lib/ccache:$PATH"
+
+exec "$@"
+EOF :mode #o755)
+
+       ;; schroot
+       (chroot:os-bootstrapped-for ,chroot-options ,chroot ,host)
+       (desc
+        ,(strcat "schroot configuration for " desc)
+        (file:contains-ini-settings
+         ,conf
+         ,@(mapcar
+            (curry #'cons (format nil "~A-~A-sbuild" suite arch))
+            `(("description" ,desc)
+              ("groups"      "root,sbuild")
+              ("root-groups" "root,sbuild")
+              ("profile"     "sbuild")
+              ("type"        "directory")
+              ("directory"   ,(drop-trailing-slash (unix-namestring chroot)))
+              ,@(and (get-hostattrs-car 'schroot:uses-overlays)
+                     `(("union-type" "overlay")))
+
+              ;; If we're building a sid chroot, add useful aliases.  In order
+              ;; to avoid more than one schroot getting the same aliases, we
+              ;; only do this if the arch of the chroot equals the host arch.
+              ,@(and (string= suite "unstable")
+                     (string= arch (os:debian-architecture
+                                    (get-hostattrs-car :os)))
+                     `(("aliases"
+                        ,(format
+                          nil "~@{~A~^,~}"
+                          "sid"
+                          ;; If the user wants to build for experimental, they
+                          ;; would use their sid chroot and sbuild's
+                          ;; --extra-repository option to enable experimental.
+                          "rc-buggy"
+                          "experimental"
+                          ;; We assume that building for UNRELEASED means
+                          ;; building for unstable.
+                          "UNRELEASED"
+                          ;; The following is for dgit compatibility.
+                          (strcat "UNRELEASED-"
+                                  (os:debian-architecture os)
+                                  "-sbuild")))))
+
+              ("command-prefix"
+               ,(if use-ccache
+                    "/var/cache/ccache-sbuild/sbuild-setup:eatmydata"
+                    "eatmydata"))))))
+
+       ;; TODO We should kill any sessions still using the chroot before
+       ;; destroying it (as suggested by sbuild-destroychroot(8)).
+       :unapply
+       (unapply (chroot:os-bootstrapped-for ,chroot-options ,chroot ,host))
+       (file:does-not-exist ,conf))))
+
+;; Here we combine Propellor's Sbuild.osDebianStandard and Sbuild.update.
+(defpropspec standard-debian-schroot :posix (&optional (period :weekly))
+  "Properties that will be wanted in almost any Debian sbuild schroot, but not
+in sbuild schroots for other operating systems.
+
+Includes replacing use of sbuild-update(1)."
+  (:desc "Standard Debian sbuild properties")
+  `(eseqprops (apt:standard-sources.list)
+              (periodic:at-most ,period "sbuild schroot updated"
+                (apt:updated) (apt:upgraded) (apt:autoremoved))))
diff --git a/src/property/schroot.lisp b/src/property/schroot.lisp
new file mode 100644
index 0000000..2e6cb40
--- /dev/null
+++ b/src/property/schroot.lisp
@@ -0,0 +1,65 @@
+;;; Consfigurator -- Lisp declarative configuration management system
+
+;;; Copyright (C) 2016, 2021  Sean Whitton <spwhitton@spwhitton.name>
+
+;;; This file is free software; you can redistribute it and/or modify
+;;; it under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3, or (at your option)
+;;; any later version.
+
+;;; This file is distributed in the hope that it will be useful,
+;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+
+;;; You should have received a copy of the GNU General Public License
+;;; along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+(in-package :consfigurator.property.schroot)
+(named-readtables:in-readtable :consfigurator)
+
+(defproplist installed :posix ()
+  "Ensure that schroot(1) is installed."
+  (:desc "schroot(1) installed")
+  (os:etypecase
+    (debianlike (apt:installed "schroot"))))
+
+(defprop uses-overlays :posix ()
+  "Indicate that schroots on a host should use 'union-type=overlay'.
+
+Adding this property does not actually ensure that the line
+'union-type=overlay' is present in any schroot config files.  See SBUILD:BUILT
+for example usage, via SCHROOT:OVERLAYS-IN-TMPFS."
+  (:desc "schroots on host use union-type=overlay")
+  (:hostattrs (push-hostattrs 'uses-overlays t)))
+
+(defprop overlays-in-tmpfs :posix ()
+  "Configure schroot(1) such that all schroots with 'union-type=overlay' in
+their configuration will run their overlays in a tmpfs.  Unapplicable, so if
+the package you are working on FTBFS when overlays are in tmpfs, you can
+toggle this off for a host, and then toggle it back on again later.
+
+Implicitly sets SCHROOT:USES-OVERLAYS.
+
+Shell script from <https://wiki.debian.org/sbuild>."
+  (:desc "schroot overlays in tmpfs")
+  (:hostattrs (push-hostattrs 'uses-overlays t))
+  (:apply (file:has-content "/etc/schroot/setup.d/04tmpfs"
+#>EOF>#!/bin/sh
+
+set -e
+
+. "$SETUP_DATA_DIR/common-data"
+. "$SETUP_DATA_DIR/common-functions"
+. "$SETUP_DATA_DIR/common-config"
+
+
+if [ $STAGE = "setup-start" ]; then
+  mount -t tmpfs overlay /var/lib/schroot/union/overlay
+elif [ $STAGE = "setup-recover" ]; then
+  mount -t tmpfs overlay /var/lib/schroot/union/overlay
+elif [ $STAGE = "setup-stop" ]; then
+  umount -f /var/lib/schroot/union/overlay
+fi
+EOF :mode #o755))
+  (:unapply (file:does-not-exist "/etc/schroot/setup.d/04tmpfs")))