Add timeout to automatic updates
This commit is contained in:
parent
46f06301f7
commit
b08c7f8abf
1 changed files with 189 additions and 186 deletions
|
|
@ -47,211 +47,214 @@ in
|
|||
|
||||
config = mkIf cfg.enable {
|
||||
email.enable = true;
|
||||
systemd.services.nixos-upgrade.script = mkOverride 50 (
|
||||
let
|
||||
nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
|
||||
nix-store = "${pkgs.nix}/bin/nix-store";
|
||||
diff = "${pkgs.diffutils}/bin/diff";
|
||||
grep = "${pkgs.gnugrep}/bin/grep";
|
||||
git = "${pkgs.git}/bin/git";
|
||||
ssh = "${pkgs.openssh}/bin/ssh";
|
||||
sudo = "${pkgs.sudo}/bin/sudo";
|
||||
shutdown = "${config.systemd.package}/bin/shutdown";
|
||||
sendmail = "${pkgs.system-sendmail}/bin/sendmail";
|
||||
upgradeFlag = optional (cfg.channel == null) "--upgrade";
|
||||
in ''
|
||||
set -o pipefail
|
||||
systemd.services.nixos-upgrade = {
|
||||
serviceConfig.TimeoutStartSec = "2h";
|
||||
script = mkOverride 50 (
|
||||
let
|
||||
nixos-rebuild = "${config.system.build.nixos-rebuild}/bin/nixos-rebuild";
|
||||
nix-store = "${pkgs.nix}/bin/nix-store";
|
||||
diff = "${pkgs.diffutils}/bin/diff";
|
||||
grep = "${pkgs.gnugrep}/bin/grep";
|
||||
git = "${pkgs.git}/bin/git";
|
||||
ssh = "${pkgs.openssh}/bin/ssh";
|
||||
sudo = "${pkgs.sudo}/bin/sudo";
|
||||
shutdown = "${config.systemd.package}/bin/shutdown";
|
||||
sendmail = "${pkgs.system-sendmail}/bin/sendmail";
|
||||
upgradeFlag = optional (cfg.channel == null) "--upgrade";
|
||||
in ''
|
||||
set -o pipefail
|
||||
|
||||
indent()
|
||||
{
|
||||
while read -r line ; do
|
||||
echo " $line"
|
||||
done <<< "$1"
|
||||
}
|
||||
|
||||
start_time="$(date)"
|
||||
reboot_allowed="no"
|
||||
activate_configuration="yes"
|
||||
do_reboot="no"
|
||||
exit_code=0
|
||||
|
||||
${optionalString cfg.allowReboot ''
|
||||
reboot_allowed="yes"
|
||||
|
||||
${optionalString (cfg.rebootWindow != null) ''
|
||||
current_time="$(${date} +%H:%M)"
|
||||
lower="${cfg.rebootWindow.lower}"
|
||||
upper="${cfg.rebootWindow.upper}"
|
||||
if [[ "''${lower}" < "''${upper}" ]]; then
|
||||
if [[ "''${current_time}" > "''${lower}" ]] && \
|
||||
[[ "''${current_time}" < "''${upper}" ]]; then
|
||||
reboot_allowed="yes"
|
||||
else
|
||||
reboot_allowed="no"
|
||||
fi
|
||||
else
|
||||
# lower > upper, so we are crossing midnight (e.g. lower=23h, upper=6h)
|
||||
# we want to reboot if cur > 23h or cur < 6h
|
||||
if [[ "''${current_time}" < "''${upper}" ]] || \
|
||||
[[ "''${current_time}" > "''${lower}" ]]; then
|
||||
reboot_allowed="yes"
|
||||
else
|
||||
reboot_allowed="no"
|
||||
fi
|
||||
fi
|
||||
''}
|
||||
''}
|
||||
|
||||
output_file="$(mktemp)"
|
||||
send_email=no
|
||||
email_subject_additions=
|
||||
|
||||
${optionalString cfg.gitPull ''
|
||||
{
|
||||
cd /etc/nixos
|
||||
echo "→ Refreshing git repository at /etc/nixos." | tee -a "$output_file"
|
||||
if ! ${optionalString (cfg.gitDeploymentKeyFile != null) ''GIT_SSH_COMMAND='${ssh} -i "${cfg.gitDeploymentKeyFile}" -o IdentitiesOnly=yes' ''}${optionalString (cfg.gitUser != null) ''${sudo} -nu ${cfg.gitUser} ''}${git} pull 2>&1 | tee -a "$output_file" ; then
|
||||
send_email=yes
|
||||
email_subject_additions="$email_subject_additions, errors during git pull"
|
||||
fi
|
||||
}
|
||||
''}
|
||||
|
||||
echo "→ Running upgrade." | tee -a "$output_file"
|
||||
${nixos-rebuild} boot ${toString (cfg.flags ++ upgradeFlag)} 2>&1 | tee -a "$output_file" || exit_code=$?
|
||||
|
||||
email_subject="Upgrade succeeded"
|
||||
email_body="The system upgrade started at $start_time has succeeded."
|
||||
if [ "$exit_code" -ne 0 ] ; then
|
||||
send_email=yes
|
||||
email_subject="Upgrade failed (exit code $exit_code)"
|
||||
email_body="The system upgrade started at $start_time has failed with exit code $exit_code."
|
||||
else
|
||||
echo "→ Determining package differences." | tee -a "$output_file"
|
||||
installed_packages()
|
||||
indent()
|
||||
{
|
||||
${nix-store} --query --requisites "$1" | cut -d- -f2- | sort | uniq
|
||||
while read -r line ; do
|
||||
echo " $line"
|
||||
done <<< "$1"
|
||||
}
|
||||
current_packages="$(installed_packages /run/current-system)"
|
||||
built_packages="$(installed_packages /nix/var/nix/profiles/system)"
|
||||
${diff} -y --suppress-common-lines --width=71 \
|
||||
<(printf "Current generation\n------------------\n%s" "$current_packages") \
|
||||
<(printf "New generation\n--------------\n%s" "$built_packages") \
|
||||
| tee -a "$output_file" || true
|
||||
|
||||
booted_version="$(readlink /run/booted-system/{initrd,kernel,kernel-modules})"
|
||||
built_version="$(readlink /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})"
|
||||
start_time="$(date)"
|
||||
reboot_allowed="no"
|
||||
activate_configuration="yes"
|
||||
do_reboot="no"
|
||||
exit_code=0
|
||||
|
||||
echo "→ Checking if a reboot is needed." | tee -a "$output_file"
|
||||
if [ "$booted_version" != "$built_version" ] ; then
|
||||
version_comparison="$(cat <<-EOF
|
||||
The booted kernel version
|
||||
$(indent "$booted_version")
|
||||
does not match the newly built
|
||||
$(indent "$built_version")
|
||||
.
|
||||
EOF
|
||||
)"
|
||||
echo "$version_comparison"
|
||||
send_email=yes
|
||||
email_body="$(cat <<-EOF
|
||||
$email_body
|
||||
${optionalString cfg.allowReboot ''
|
||||
reboot_allowed="yes"
|
||||
|
||||
|
||||
A reboot is required, because:
|
||||
------------------------------
|
||||
$version_comparison
|
||||
EOF
|
||||
)"
|
||||
activate_configuration="no"
|
||||
|
||||
if [ "$reboot_allowed" = "yes" ] ; then
|
||||
echo "→ Checking if a reboot is allowed." | tee -a "$output_file"
|
||||
|
||||
# Check if any user sessions are open
|
||||
active_users=$(users | tr ' ' '\n' | sort | uniq | ${grep} -vE '^(${concatStringsSep "|" cfg.rebootIgnoreUsersActive})$' || true)
|
||||
if [ -n "$active_users" ] ; then
|
||||
reboot_allowed=no
|
||||
email_body="$(printf "%s\n%s\n%s" "$email_body" "The system cannot reboot since the following users are active:" "$active_users")"
|
||||
echo "$(echo $active_users | wc -l) active users blocking reboot." | tee -a "$output_file"
|
||||
fi
|
||||
|
||||
${optionalString config.virtualisation.libvirtd.enable ''
|
||||
# Check if virtual machines are running
|
||||
active_vms=$(${pkgs.libvirt}/bin/virsh list --state-running --no-autostart --id --name || true)
|
||||
if [ -n "$active_vms" ] ; then
|
||||
reboot_allowed=no
|
||||
email_body="$(printf "%s\n%s\n%s" "$email_body" "The system cannot reboot since the following virtual machines are active:" "$active_vms")"
|
||||
echo "$(echo $active_vms | wc -l) active VMs blocking reboot." | tee -a "$output_file"
|
||||
fi
|
||||
''}
|
||||
fi
|
||||
|
||||
if [ "$reboot_allowed" = "yes" ] && [ $exit_code -eq 0 ] ; then
|
||||
email_body="$(printf "%s\n%s" "$email_body" "The system will reboot now.")"
|
||||
do_reboot="yes"
|
||||
activate_configuration="yes"
|
||||
email_subject_additions="$email_subject_additions, system will reboot"
|
||||
${optionalString (cfg.rebootWindow != null) ''
|
||||
current_time="$(${date} +%H:%M)"
|
||||
lower="${cfg.rebootWindow.lower}"
|
||||
upper="${cfg.rebootWindow.upper}"
|
||||
if [[ "''${lower}" < "''${upper}" ]]; then
|
||||
if [[ "''${current_time}" > "''${lower}" ]] && \
|
||||
[[ "''${current_time}" < "''${upper}" ]]; then
|
||||
reboot_allowed="yes"
|
||||
else
|
||||
email_body="$(printf "%s\n%s" "$email_body" "The upgraded configuration will be activated on the next reboot.")"
|
||||
email_subject_additions="$email_subject_additions, reboot required"
|
||||
reboot_allowed="no"
|
||||
fi
|
||||
else
|
||||
# lower > upper, so we are crossing midnight (e.g. lower=23h, upper=6h)
|
||||
# we want to reboot if cur > 23h or cur < 6h
|
||||
if [[ "''${current_time}" < "''${upper}" ]] || \
|
||||
[[ "''${current_time}" > "''${lower}" ]]; then
|
||||
reboot_allowed="yes"
|
||||
else
|
||||
reboot_allowed="no"
|
||||
fi
|
||||
fi
|
||||
|
||||
${optionalString (cfg.operation == "switch") ''
|
||||
if [ "$activate_configuration" = "yes" ] ; then
|
||||
echo "→ Activating new configuration." | tee -a "$output_file"
|
||||
${nixos-rebuild} switch ${toString cfg.flags} 2>&1 | tee -a "$output_file" || exit_code=$?
|
||||
fi
|
||||
''}
|
||||
fi
|
||||
''}
|
||||
|
||||
upgrade_output="$(cat "$output_file")"
|
||||
rm -f "$output_file"
|
||||
output_file="$(mktemp)"
|
||||
send_email=no
|
||||
email_subject_additions=
|
||||
|
||||
possible_warnings="$(${grep} -e "^\(warning\|trace\|evaluation warning\):" <<<"$upgrade_output" || true)"
|
||||
if [ "$possible_warnings" != "" ] ; then
|
||||
send_email=yes
|
||||
email_subject_additions="$email_subject_additions with warnings"
|
||||
email_body="$(cat <<-EOF
|
||||
$email_body
|
||||
${optionalString cfg.gitPull ''
|
||||
{
|
||||
cd /etc/nixos
|
||||
echo "→ Refreshing git repository at /etc/nixos." | tee -a "$output_file"
|
||||
if ! ${optionalString (cfg.gitDeploymentKeyFile != null) ''GIT_SSH_COMMAND='${ssh} -i "${cfg.gitDeploymentKeyFile}" -o IdentitiesOnly=yes' ''}${optionalString (cfg.gitUser != null) ''${sudo} -nu ${cfg.gitUser} ''}${git} pull 2>&1 | tee -a "$output_file" ; then
|
||||
send_email=yes
|
||||
email_subject_additions="$email_subject_additions, errors during git pull"
|
||||
fi
|
||||
}
|
||||
''}
|
||||
|
||||
echo "→ Running upgrade." | tee -a "$output_file"
|
||||
${nixos-rebuild} boot ${toString (cfg.flags ++ upgradeFlag)} 2>&1 | tee -a "$output_file" || exit_code=$?
|
||||
|
||||
email_subject="Upgrade succeeded"
|
||||
email_body="The system upgrade started at $start_time has succeeded."
|
||||
if [ "$exit_code" -ne 0 ] ; then
|
||||
send_email=yes
|
||||
email_subject="Upgrade failed (exit code $exit_code)"
|
||||
email_body="The system upgrade started at $start_time has failed with exit code $exit_code."
|
||||
else
|
||||
echo "→ Determining package differences." | tee -a "$output_file"
|
||||
installed_packages()
|
||||
{
|
||||
${nix-store} --query --requisites "$1" | cut -d- -f2- | sort | uniq
|
||||
}
|
||||
current_packages="$(installed_packages /run/current-system)"
|
||||
built_packages="$(installed_packages /nix/var/nix/profiles/system)"
|
||||
${diff} -y --suppress-common-lines --width=71 \
|
||||
<(printf "Current generation\n------------------\n%s" "$current_packages") \
|
||||
<(printf "New generation\n--------------\n%s" "$built_packages") \
|
||||
| tee -a "$output_file" || true
|
||||
|
||||
booted_version="$(readlink /run/booted-system/{initrd,kernel,kernel-modules})"
|
||||
built_version="$(readlink /nix/var/nix/profiles/system/{initrd,kernel,kernel-modules})"
|
||||
|
||||
echo "→ Checking if a reboot is needed." | tee -a "$output_file"
|
||||
if [ "$booted_version" != "$built_version" ] ; then
|
||||
version_comparison="$(cat <<-EOF
|
||||
The booted kernel version
|
||||
$(indent "$booted_version")
|
||||
does not match the newly built
|
||||
$(indent "$built_version")
|
||||
.
|
||||
EOF
|
||||
)"
|
||||
echo "$version_comparison"
|
||||
send_email=yes
|
||||
email_body="$(cat <<-EOF
|
||||
$email_body
|
||||
|
||||
|
||||
These trace messages and warnings were encountered:
|
||||
---------------------------------------------------
|
||||
$possible_warnings
|
||||
EOF
|
||||
)"
|
||||
fi
|
||||
A reboot is required, because:
|
||||
------------------------------
|
||||
$version_comparison
|
||||
EOF
|
||||
)"
|
||||
activate_configuration="no"
|
||||
|
||||
${optionalString cfg.sendEmail ''
|
||||
if [ "$send_email" = "yes" ] ; then
|
||||
echo "→ Sending e-mail to ${toAddress}."
|
||||
${sendmail} -t -X - <<-EOF
|
||||
To: ${toAddress}
|
||||
From: ${fromIdentity}
|
||||
Subject: $email_subject$email_subject_additions
|
||||
Content-Transfer-Encoding: 8bit
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
X-Priority: 3
|
||||
if [ "$reboot_allowed" = "yes" ] ; then
|
||||
echo "→ Checking if a reboot is allowed." | tee -a "$output_file"
|
||||
|
||||
$email_body
|
||||
# Check if any user sessions are open
|
||||
active_users=$(users | tr ' ' '\n' | sort | uniq | ${grep} -vE '^(${concatStringsSep "|" cfg.rebootIgnoreUsersActive})$' || true)
|
||||
if [ -n "$active_users" ] ; then
|
||||
reboot_allowed=no
|
||||
email_body="$(printf "%s\n%s\n%s" "$email_body" "The system cannot reboot since the following users are active:" "$active_users")"
|
||||
echo "$(echo $active_users | wc -l) active users blocking reboot." | tee -a "$output_file"
|
||||
fi
|
||||
|
||||
${optionalString config.virtualisation.libvirtd.enable ''
|
||||
# Check if virtual machines are running
|
||||
active_vms=$(${pkgs.libvirt}/bin/virsh list --state-running --no-autostart --id --name || true)
|
||||
if [ -n "$active_vms" ] ; then
|
||||
reboot_allowed=no
|
||||
email_body="$(printf "%s\n%s\n%s" "$email_body" "The system cannot reboot since the following virtual machines are active:" "$active_vms")"
|
||||
echo "$(echo $active_vms | wc -l) active VMs blocking reboot." | tee -a "$output_file"
|
||||
fi
|
||||
''}
|
||||
fi
|
||||
|
||||
if [ "$reboot_allowed" = "yes" ] && [ $exit_code -eq 0 ] ; then
|
||||
email_body="$(printf "%s\n%s" "$email_body" "The system will reboot now.")"
|
||||
do_reboot="yes"
|
||||
activate_configuration="yes"
|
||||
email_subject_additions="$email_subject_additions, system will reboot"
|
||||
else
|
||||
email_body="$(printf "%s\n%s" "$email_body" "The upgraded configuration will be activated on the next reboot.")"
|
||||
email_subject_additions="$email_subject_additions, reboot required"
|
||||
fi
|
||||
fi
|
||||
|
||||
${optionalString (cfg.operation == "switch") ''
|
||||
if [ "$activate_configuration" = "yes" ] ; then
|
||||
echo "→ Activating new configuration." | tee -a "$output_file"
|
||||
${nixos-rebuild} switch ${toString cfg.flags} 2>&1 | tee -a "$output_file" || exit_code=$?
|
||||
fi
|
||||
''}
|
||||
fi
|
||||
|
||||
upgrade_output="$(cat "$output_file")"
|
||||
rm -f "$output_file"
|
||||
|
||||
possible_warnings="$(${grep} -e "^\(warning\|trace\|evaluation warning\):" <<<"$upgrade_output" || true)"
|
||||
if [ "$possible_warnings" != "" ] ; then
|
||||
send_email=yes
|
||||
email_subject_additions="$email_subject_additions with warnings"
|
||||
email_body="$(cat <<-EOF
|
||||
$email_body
|
||||
|
||||
|
||||
Full upgrade output:
|
||||
--------------------
|
||||
$upgrade_output
|
||||
EOF
|
||||
fi
|
||||
''}
|
||||
These trace messages and warnings were encountered:
|
||||
---------------------------------------------------
|
||||
$possible_warnings
|
||||
EOF
|
||||
)"
|
||||
fi
|
||||
|
||||
if [ "$do_reboot" = "yes" ] ; then
|
||||
echo "→ Rebooting system."
|
||||
${shutdown} -r +1
|
||||
fi
|
||||
${optionalString cfg.sendEmail ''
|
||||
if [ "$send_email" = "yes" ] ; then
|
||||
echo "→ Sending e-mail to ${toAddress}."
|
||||
${sendmail} -t -X - <<-EOF
|
||||
To: ${toAddress}
|
||||
From: ${fromIdentity}
|
||||
Subject: $email_subject$email_subject_additions
|
||||
Content-Transfer-Encoding: 8bit
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
X-Priority: 3
|
||||
|
||||
exit $exit_code
|
||||
''
|
||||
);
|
||||
$email_body
|
||||
|
||||
|
||||
Full upgrade output:
|
||||
--------------------
|
||||
$upgrade_output
|
||||
EOF
|
||||
fi
|
||||
''}
|
||||
|
||||
if [ "$do_reboot" = "yes" ] ; then
|
||||
echo "→ Rebooting system."
|
||||
${shutdown} -r +1
|
||||
fi
|
||||
|
||||
exit $exit_code
|
||||
''
|
||||
);
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue