From b08c7f8abfde1ab58fccbdb3fd3c03e70b7fc9b8 Mon Sep 17 00:00:00 2001 From: fruchti Date: Mon, 19 Jan 2026 21:11:01 +0100 Subject: [PATCH] Add timeout to automatic updates --- options/auto-upgrade.nix | 375 ++++++++++++++++++++------------------- 1 file changed, 189 insertions(+), 186 deletions(-) diff --git a/options/auto-upgrade.nix b/options/auto-upgrade.nix index e6e782e..80f83a9 100644 --- a/options/auto-upgrade.nix +++ b/options/auto-upgrade.nix @@ -47,211 +47,214 @@ in config = mkIf cfg.enable { email.enable = true; - systemd.services.nixos-upgrade.script = mkOverride 50 ( - let - nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild"; - nix-store = "${pkgs.nix}/bin/nix-store"; - diff = "${pkgs.diffutils}/bin/diff"; - grep = "${pkgs.gnugrep}/bin/grep"; - git = "${pkgs.git}/bin/git"; - ssh = "${pkgs.openssh}/bin/ssh"; - sudo = "${pkgs.sudo}/bin/sudo"; - shutdown = "${config.systemd.package}/bin/shutdown"; - sendmail = "${pkgs.system-sendmail}/bin/sendmail"; - upgradeFlag = optional (cfg.channel == null) "--upgrade"; - in '' - set -o pipefail + systemd.services.nixos-upgrade = { + serviceConfig.TimeoutStartSec = "2h"; + script = mkOverride 50 ( + let + nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild"; + nix-store = "${pkgs.nix}/bin/nix-store"; + diff = "${pkgs.diffutils}/bin/diff"; + grep = "${pkgs.gnugrep}/bin/grep"; + git = "${pkgs.git}/bin/git"; + ssh = "${pkgs.openssh}/bin/ssh"; + sudo = "${pkgs.sudo}/bin/sudo"; + shutdown = "${config.systemd.package}/bin/shutdown"; + sendmail = "${pkgs.system-sendmail}/bin/sendmail"; + upgradeFlag = optional (cfg.channel == null) "--upgrade"; + in '' + set -o pipefail - indent() - { - while read -r line ; do - echo " $line" - done <<< "$1" - } - - start_time="$(date)" - reboot_allowed="no" - activate_configuration="yes" - do_reboot="no" - exit_code=0 - - ${optionalString cfg.allowReboot '' - reboot_allowed="yes" - - ${optionalString (cfg.rebootWindow != null) '' - current_time="$(${date} +%H:%M)" - lower="${cfg.rebootWindow.lower}" - upper="${cfg.rebootWindow.upper}" - if [[ "''${lower}" < "''${upper}" ]]; then - if [[ "''${current_time}" > "''${lower}" ]] && \ - [[ "''${current_time}" < "''${upper}" ]]; then - reboot_allowed="yes" - else - reboot_allowed="no" - fi - else - # lower > upper, so we are crossing midnight (e.g. lower=23h, upper=6h) - # we want to reboot if cur > 23h or cur < 6h - if [[ "''${current_time}" < "''${upper}" ]] || \ - [[ "''${current_time}" > "''${lower}" ]]; then - reboot_allowed="yes" - else - reboot_allowed="no" - fi - fi - ''} - ''} - - output_file="$(mktemp)" - send_email=no - email_subject_additions= - - ${optionalString cfg.gitPull '' - { - cd /etc/nixos - echo "→ Refreshing git repository at /etc/nixos." | tee -a "$output_file" - if ! ${optionalString (cfg.gitDeploymentKeyFile != null) ''GIT_SSH_COMMAND='${ssh} -i "${cfg.gitDeploymentKeyFile}" -o IdentitiesOnly=yes' ''}${optionalString (cfg.gitUser != null) ''${sudo} -nu ${cfg.gitUser} ''}${git} pull 2>&1 | tee -a "$output_file" ; then - send_email=yes - email_subject_additions="$email_subject_additions, errors during git pull" - fi - } - ''} - - echo "→ Running upgrade." | tee -a "$output_file" - ${nixos-rebuild} boot ${toString (cfg.flags ++ upgradeFlag)} 2>&1 | tee -a "$output_file" || exit_code=$? - - email_subject="Upgrade succeeded" - email_body="The system upgrade started at $start_time has succeeded." - if [ "$exit_code" -ne 0 ] ; then - send_email=yes - email_subject="Upgrade failed (exit code $exit_code)" - email_body="The system upgrade started at $start_time has failed with exit code $exit_code." - else - echo "→ Determining package differences." | tee -a "$output_file" - installed_packages() + indent() { - ${nix-store} --query --requisites "$1" | cut -d- -f2- | sort | uniq + while read -r line ; do + echo " $line" + done <<< "$1" } - current_packages="$(installed_packages /run/current-system)" - built_packages="$(installed_packages /nix/var/nix/profiles/system)" - ${diff} -y --suppress-common-lines --width=71 \ - <(printf "Current generation\n------------------\n%s" "$current_packages") \ - <(printf "New generation\n--------------\n%s" "$built_packages") \ - | tee -a "$output_file" || true - booted_version="$(readlink /run/booted-system/{initrd,kernel,kernel-modules})" - built_version="$(readlink /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})" + start_time="$(date)" + reboot_allowed="no" + activate_configuration="yes" + do_reboot="no" + exit_code=0 - echo "→ Checking if a reboot is needed." | tee -a "$output_file" - if [ "$booted_version" != "$built_version" ] ; then - version_comparison="$(cat <<-EOF - The booted kernel version - $(indent "$booted_version") - does not match the newly built - $(indent "$built_version") - . - EOF - )" - echo "$version_comparison" - send_email=yes - email_body="$(cat <<-EOF - $email_body + ${optionalString cfg.allowReboot '' + reboot_allowed="yes" - - A reboot is required, because: - ------------------------------ - $version_comparison - EOF - )" - activate_configuration="no" - - if [ "$reboot_allowed" = "yes" ] ; then - echo "→ Checking if a reboot is allowed." | tee -a "$output_file" - - # Check if any user sessions are open - active_users=$(users | tr ' ' '\n' | sort | uniq | ${grep} -vE '^(${concatStringsSep "|" cfg.rebootIgnoreUsersActive})$' || true) - if [ -n "$active_users" ] ; then - reboot_allowed=no - email_body="$(printf "%s\n%s\n%s" "$email_body" "The system cannot reboot since the following users are active:" "$active_users")" - echo "$(echo $active_users | wc -l) active users blocking reboot." | tee -a "$output_file" - fi - - ${optionalString config.virtualisation.libvirtd.enable '' - # Check if virtual machines are running - active_vms=$(${pkgs.libvirt}/bin/virsh list --state-running --no-autostart --id --name || true) - if [ -n "$active_vms" ] ; then - reboot_allowed=no - email_body="$(printf "%s\n%s\n%s" "$email_body" "The system cannot reboot since the following virtual machines are active:" "$active_vms")" - echo "$(echo $active_vms | wc -l) active VMs blocking reboot." | tee -a "$output_file" - fi - ''} - fi - - if [ "$reboot_allowed" = "yes" ] && [ $exit_code -eq 0 ] ; then - email_body="$(printf "%s\n%s" "$email_body" "The system will reboot now.")" - do_reboot="yes" - activate_configuration="yes" - email_subject_additions="$email_subject_additions, system will reboot" + ${optionalString (cfg.rebootWindow != null) '' + current_time="$(${date} +%H:%M)" + lower="${cfg.rebootWindow.lower}" + upper="${cfg.rebootWindow.upper}" + if [[ "''${lower}" < "''${upper}" ]]; then + if [[ "''${current_time}" > "''${lower}" ]] && \ + [[ "''${current_time}" < "''${upper}" ]]; then + reboot_allowed="yes" else - email_body="$(printf "%s\n%s" "$email_body" "The upgraded configuration will be activated on the next reboot.")" - email_subject_additions="$email_subject_additions, reboot required" + reboot_allowed="no" + fi + else + # lower > upper, so we are crossing midnight (e.g. lower=23h, upper=6h) + # we want to reboot if cur > 23h or cur < 6h + if [[ "''${current_time}" < "''${upper}" ]] || \ + [[ "''${current_time}" > "''${lower}" ]]; then + reboot_allowed="yes" + else + reboot_allowed="no" fi - fi - - ${optionalString (cfg.operation == "switch") '' - if [ "$activate_configuration" = "yes" ] ; then - echo "→ Activating new configuration." | tee -a "$output_file" - ${nixos-rebuild} switch ${toString cfg.flags} 2>&1 | tee -a "$output_file" || exit_code=$? fi ''} - fi + ''} - upgrade_output="$(cat "$output_file")" - rm -f "$output_file" + output_file="$(mktemp)" + send_email=no + email_subject_additions= - possible_warnings="$(${grep} -e "^\(warning\|trace\|evaluation warning\):" <<<"$upgrade_output" || true)" - if [ "$possible_warnings" != "" ] ; then - send_email=yes - email_subject_additions="$email_subject_additions with warnings" - email_body="$(cat <<-EOF - $email_body + ${optionalString cfg.gitPull '' + { + cd /etc/nixos + echo "→ Refreshing git repository at /etc/nixos." | tee -a "$output_file" + if ! ${optionalString (cfg.gitDeploymentKeyFile != null) ''GIT_SSH_COMMAND='${ssh} -i "${cfg.gitDeploymentKeyFile}" -o IdentitiesOnly=yes' ''}${optionalString (cfg.gitUser != null) ''${sudo} -nu ${cfg.gitUser} ''}${git} pull 2>&1 | tee -a "$output_file" ; then + send_email=yes + email_subject_additions="$email_subject_additions, errors during git pull" + fi + } + ''} + + echo "→ Running upgrade." | tee -a "$output_file" + ${nixos-rebuild} boot ${toString (cfg.flags ++ upgradeFlag)} 2>&1 | tee -a "$output_file" || exit_code=$? + + email_subject="Upgrade succeeded" + email_body="The system upgrade started at $start_time has succeeded." + if [ "$exit_code" -ne 0 ] ; then + send_email=yes + email_subject="Upgrade failed (exit code $exit_code)" + email_body="The system upgrade started at $start_time has failed with exit code $exit_code." + else + echo "→ Determining package differences." | tee -a "$output_file" + installed_packages() + { + ${nix-store} --query --requisites "$1" | cut -d- -f2- | sort | uniq + } + current_packages="$(installed_packages /run/current-system)" + built_packages="$(installed_packages /nix/var/nix/profiles/system)" + ${diff} -y --suppress-common-lines --width=71 \ + <(printf "Current generation\n------------------\n%s" "$current_packages") \ + <(printf "New generation\n--------------\n%s" "$built_packages") \ + | tee -a "$output_file" || true + + booted_version="$(readlink /run/booted-system/{initrd,kernel,kernel-modules})" + built_version="$(readlink /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})" + + echo "→ Checking if a reboot is needed." | tee -a "$output_file" + if [ "$booted_version" != "$built_version" ] ; then + version_comparison="$(cat <<-EOF + The booted kernel version + $(indent "$booted_version") + does not match the newly built + $(indent "$built_version") + . + EOF + )" + echo "$version_comparison" + send_email=yes + email_body="$(cat <<-EOF + $email_body - These trace messages and warnings were encountered: - --------------------------------------------------- - $possible_warnings - EOF - )" - fi + A reboot is required, because: + ------------------------------ + $version_comparison + EOF + )" + activate_configuration="no" - ${optionalString cfg.sendEmail '' - if [ "$send_email" = "yes" ] ; then - echo "→ Sending e-mail to ${toAddress}." - ${sendmail} -t -X - <<-EOF - To: ${toAddress} - From: ${fromIdentity} - Subject: $email_subject$email_subject_additions - Content-Transfer-Encoding: 8bit - Content-Type: text/plain; charset=UTF-8 - X-Priority: 3 + if [ "$reboot_allowed" = "yes" ] ; then + echo "→ Checking if a reboot is allowed." | tee -a "$output_file" - $email_body + # Check if any user sessions are open + active_users=$(users | tr ' ' '\n' | sort | uniq | ${grep} -vE '^(${concatStringsSep "|" cfg.rebootIgnoreUsersActive})$' || true) + if [ -n "$active_users" ] ; then + reboot_allowed=no + email_body="$(printf "%s\n%s\n%s" "$email_body" "The system cannot reboot since the following users are active:" "$active_users")" + echo "$(echo $active_users | wc -l) active users blocking reboot." | tee -a "$output_file" + fi + + ${optionalString config.virtualisation.libvirtd.enable '' + # Check if virtual machines are running + active_vms=$(${pkgs.libvirt}/bin/virsh list --state-running --no-autostart --id --name || true) + if [ -n "$active_vms" ] ; then + reboot_allowed=no + email_body="$(printf "%s\n%s\n%s" "$email_body" "The system cannot reboot since the following virtual machines are active:" "$active_vms")" + echo "$(echo $active_vms | wc -l) active VMs blocking reboot." | tee -a "$output_file" + fi + ''} + fi + + if [ "$reboot_allowed" = "yes" ] && [ $exit_code -eq 0 ] ; then + email_body="$(printf "%s\n%s" "$email_body" "The system will reboot now.")" + do_reboot="yes" + activate_configuration="yes" + email_subject_additions="$email_subject_additions, system will reboot" + else + email_body="$(printf "%s\n%s" "$email_body" "The upgraded configuration will be activated on the next reboot.")" + email_subject_additions="$email_subject_additions, reboot required" + fi + fi + + ${optionalString (cfg.operation == "switch") '' + if [ "$activate_configuration" = "yes" ] ; then + echo "→ Activating new configuration." | tee -a "$output_file" + ${nixos-rebuild} switch ${toString cfg.flags} 2>&1 | tee -a "$output_file" || exit_code=$? + fi + ''} + fi + + upgrade_output="$(cat "$output_file")" + rm -f "$output_file" + + possible_warnings="$(${grep} -e "^\(warning\|trace\|evaluation warning\):" <<<"$upgrade_output" || true)" + if [ "$possible_warnings" != "" ] ; then + send_email=yes + email_subject_additions="$email_subject_additions with warnings" + email_body="$(cat <<-EOF + $email_body - Full upgrade output: - -------------------- - $upgrade_output - EOF - fi - ''} + These trace messages and warnings were encountered: + --------------------------------------------------- + $possible_warnings + EOF + )" + fi - if [ "$do_reboot" = "yes" ] ; then - echo "→ Rebooting system." - ${shutdown} -r +1 - fi + ${optionalString cfg.sendEmail '' + if [ "$send_email" = "yes" ] ; then + echo "→ Sending e-mail to ${toAddress}." + ${sendmail} -t -X - <<-EOF + To: ${toAddress} + From: ${fromIdentity} + Subject: $email_subject$email_subject_additions + Content-Transfer-Encoding: 8bit + Content-Type: text/plain; charset=UTF-8 + X-Priority: 3 - exit $exit_code - '' - ); + $email_body + + + Full upgrade output: + -------------------- + $upgrade_output + EOF + fi + ''} + + if [ "$do_reboot" = "yes" ] ; then + echo "→ Rebooting system." + ${shutdown} -r +1 + fi + + exit $exit_code + '' + ); + }; }; }