hosts/options/auto-upgrade.nix

{ pkgs, lib, config, ... }:
with lib;
let
    fromAddress = config.email.fromAddress;
    fromIdentity = config.email.fromIdentity;
    toAddress = config.email.adminEmail;
    cfg = config.system.autoUpgrade;
in
{
    options.system.autoUpgrade = {
        sendEmail = mkOption {
            type = types.bool;
            default = false;
            description = mdDoc ''
                Whether to send a status email after an upgrade.
            '';
        };
        gitPull = mkOption {
            type = types.bool;
            default = false;
            description = mdDoc ''
                Whether to run `git pull` in /etc/nixos before starting the upgrade.
            '';
        };
        gitDeploymentKeyFile = mkOption {
            type = types.nullOr types.str;
            default = null;
            description = mdDoc ''
                Private SSH key used for the `git pull` operation (if `gitPull` is enabled).
            '';
        };
        gitUser = mkOption {
            type = types.nullOr types.str;
            default = null;
            description = mdDoc ''
                User used for the `git pull` operation (if `gitPull` is enabled).
            '';
        };
    };

    config = mkIf cfg.enable {
        email.enable = true;
        systemd.services.nixos-upgrade.script = mkOverride 50 (
            let
                nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
                date = "${pkgs.coreutils}/bin/date";
                readlink = "${pkgs.coreutils}/bin/readlink";
                grep = "${pkgs.gnugrep}/bin/grep";
                git = "${pkgs.git}/bin/git";
                ssh = "${pkgs.openssh}/bin/ssh";
                sudo = "${pkgs.sudo}/bin/sudo";
                shutdown = "${config.systemd.package}/bin/shutdown";
                sendmail = "${pkgs.system-sendmail}/bin/sendmail";
                upgradeFlag = optional (cfg.channel == null) "--upgrade";
            in ''
                set -o pipefail

                indent()
                {
                    while read -r line ; do
                        echo "  $line"
                    done <<< "$1"
                }

                start_time="$(${date})"
                reboot_allowed="no"
                activate_configuration="yes"
                do_reboot="no"
                exit_code=0

                ${optionalString cfg.allowReboot ''
                reboot_allowed="yes"

                ${optionalString (cfg.rebootWindow != null) ''
                current_time="$(${date} +%H:%M)"
                lower="${cfg.rebootWindow.lower}"
                upper="${cfg.rebootWindow.upper}"
                if [[ "''${lower}" < "''${upper}" ]]; then
                    if [[ "''${current_time}" > "''${lower}" ]] && \
                        [[ "''${current_time}" < "''${upper}" ]]; then
                        reboot_allowed="yes"
                    else
                        reboot_allowed="no"
                    fi
                else
                    # lower > upper, so we are crossing midnight (e.g. lower=23h, upper=6h)
                    # we want to reboot if cur > 23h or cur < 6h
                    if [[ "''${current_time}" < "''${upper}" ]] || \
                        [[ "''${current_time}" > "''${lower}" ]]; then
                        reboot_allowed="yes"
                    else
                        reboot_allowed="no"
                    fi
                fi
                ''}
                ''}

                output_file="$(mktemp)"
                send_email=no
                email_subject_additions=

                ${optionalString cfg.gitPull ''
                {
                    cd /etc/nixos
                    echo "→ Refreshing git repository at /etc/nixos." | tee -a "$output_file"
                    if ! ${optionalString (cfg.gitDeploymentKeyFile != null) ''GIT_SSH_COMMAND='${ssh} -i "${cfg.gitDeploymentKeyFile}" -o IdentitiesOnly=yes' ''}${optionalString (cfg.gitUser != null) ''${sudo} -nu ${cfg.gitUser} ''}${git} pull 2>&1 | tee -a "$output_file" ; then
                        send_email=yes
                        email_subject_additions="$email_subject_additions, errors during git pull"
                    fi
                }
                ''}

                echo "→ Running upgrade." | tee -a "$output_file"
                ${nixos-rebuild} boot ${toString (cfg.flags ++ upgradeFlag)} 2>&1 | tee -a "$output_file" || exit_code=$?

                email_subject="Upgrade succeeded"
                email_body="The system upgrade started at $start_time has succeeded."
                if [ "$exit_code" -ne 0 ] ; then
                    send_email=yes
                    email_subject="Upgrade failed (exit code $exit_code)"
                    email_body="The system upgrade started at $start_time has failed with exit code $exit_code."
                fi

                booted_version="$(${readlink} /run/booted-system/{initrd,kernel,kernel-modules})"
                built_version="$(${readlink} /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})"

                echo "→ Checking if a reboot is needed." | tee -a "$output_file"
                if [ "$booted_version" != "$built_version" ] ; then
                    version_comparison="$(cat <<-EOF
                The booted kernel version
                $(indent "$booted_version")
                does not match the newly built
                $(indent "$built_version")
                .
                EOF
                    )"
                    echo "$version_comparison"
                    send_email=yes
                    email_body="$(cat <<-EOF
                $email_body


                A reboot is required, because:
                ------------------------------
                $version_comparison
                EOF
                    )"
                    activate_configuration="no"
                    if [ "$reboot_allowed" = "yes" ] && [ $exit_code -eq 0 ] ; then
                        email_body="$(printf "%s\n%s" "$email_body" "The system will reboot now.")"
                        do_reboot="yes"
                        activate_configuration="yes"
                        email_subject_additions="$email_subject_additions, system will reboot"
                    else
                        email_body="$(printf "%s\n%s" "$email_body" "The upgraded configuration will be activated on the next reboot.")"
                        email_subject_additions="$email_subject_additions, reboot required"
                    fi
                fi

                ${optionalString (cfg.operation == "switch") ''
                if [ "$activate_configuration" = "yes" ] ; then
                    echo "→ Activating new configuration." | tee -a "$output_file"
                    ${nixos-rebuild} switch ${toString cfg.flags} 2>&1 | tee -a "$output_file" || exit_code=$?
                fi
                ''}

                upgrade_output="$(cat "$output_file")"
                rm -f "$output_file"

                possible_warnings="$(${grep} -e "^\(warning\|trace\):" <<<"$upgrade_output" || true)"
                if [ "$possible_warnings" != "" ] ; then
                    send_email=yes
                    email_subject_additions="$email_subject_additions with warnings"
                    email_body="$(cat <<-EOF
                $email_body


                These trace messages and warnings were encountered:
                ---------------------------------------------------
                $possible_warnings
                EOF
                    )"
                fi

                ${optionalString cfg.sendEmail ''
                if [ "$send_email" = "yes" ] ; then
                    echo "→ Sending e-mail to ${toAddress}."
                    ${sendmail} -t -X - <<-EOF
                To: ${toAddress}
                From: ${fromIdentity}
                Subject: $email_subject$email_subject_additions
                Content-Transfer-Encoding: 8bit
                Content-Type: text/plain; charset=UTF-8
                X-Priority: 3

                $email_body


                Full upgrade output:
                --------------------
                $upgrade_output
                EOF
                fi
                ''}

                if [ "$do_reboot" = "yes" ] ; then
                    echo "→ Rebooting system."
                    ${shutdown} -r +1
                fi

                exit $exit_code
            ''
        );
    };
}