Health check

root@lab:~/scripts# df --output=pcent / | tail -n 1
  5%
root@lab:~/scripts# grep -E '^Mem(Total|Available):' /proc/meminfo
MemTotal:        8025128 kB
MemAvailable:    7280516 kB
root@lab:~/scripts# cat /proc/loadavg; nproc
0.36 0.24 0.16 1/307 23
8
root@lab:~/scripts# pgrep -cx sleep
1
root@lab:~/scripts# pgrep -cx cron
1
root@lab:~/scripts# pgrep -cx nginx; echo "exit $?"
0
exit 1

pgrep exits 1 when nothing matches. Under set -e that would stop the script, so the finished script writes pgrep -cx -- "$name" || true.

The load sum, with the 10# guard:

root@lab:~/scripts# load=0.36; cpus=8
root@lab:~/scripts# whole=${load%.*}; frac=${load#*.}
root@lab:~/scripts# echo $(( (10#$whole * 100 + 10#$frac) / cpus ))
4

That is 4 percent of the machine's capacity. 100 percent means every CPU is busy on average.

This is how record and one check look. The full script is in the next stage.

record() {
  local status=$1 check=$2 detail=$3 code
  case $status in
    OK)   code=0 ;;
    WARN) code=1 ;;
    *)    code=2 ;;
  esac
  (( code > worst )) && worst=$code
  results+=("$(printf '%-6s %-20s %s' "$status" "$check" "$detail")")
}

Storing finished lines in an array means the report can be printed to the screen and written to a file without running the checks twice.

(( code > worst )) && worst=$code returns a non-zero status when the test is false, but set -e ignores failures on the left of &&, so it is safe here. It would not be safe as the last line of the function, because the function would then return that status.

Save this as health.conf:

# Settings for health-check.sh
DISK_WARN=70
DISK_FAIL=90
MEM_WARN=25
MEM_FAIL=10
LOAD_WARN=100
LOAD_FAIL=200
MOUNTS=/ /root/scripts
PROCESSES=sleep cron

And this is the whole script, health-check.sh:

#!/bin/bash
# Checks disk use, available memory, load and named processes.
# Exits 0 when all is well, 1 on a warning, 2 on a failure and 3 on a usage error.
set -euo pipefail

readonly SCRIPT_NAME=${0##*/}

# Defaults. A config file (-c) changes them, and -s changes them again.
disk_warn=80     # percent used
disk_fail=90
mem_warn=20      # percent available
mem_fail=10
load_warn=100    # 1-minute load as a percentage of the CPU count
load_fail=200
mounts=(/)
processes=()

report_file=""
results=()
worst=0

usage() {
  cat <<USAGE
Usage: $SCRIPT_NAME [-c FILE] [-s KEY=VALUE]... [-d MOUNT]... [-p NAME]... [-o FILE]
  -c FILE       read settings from FILE (KEY=VALUE lines)
  -s KEY=VALUE  set one setting, overriding the config file
  -d MOUNT      check disk use of MOUNT (repeatable, default /)
  -p NAME       check that a process called NAME is running (repeatable)
  -o FILE       also write the report to FILE
  -h            show this help
Keys: DISK_WARN DISK_FAIL MEM_WARN MEM_FAIL LOAD_WARN LOAD_FAIL MOUNTS PROCESSES
USAGE
}

log() {
  printf '%s %s: %s\n' "$(date '+%F %T')" "$SCRIPT_NAME" "$*" >&2
}

die() {
  log "ERROR: $*"
  exit 3
}

is_number() {
  [[ $1 =~ ^[0-9]+$ ]]
}

# Applies one KEY=VALUE setting from the config file or -s.
set_setting() {
  local key=$1 value=$2
  case $key in
    DISK_WARN|DISK_FAIL|MEM_WARN|MEM_FAIL|LOAD_WARN|LOAD_FAIL)
      is_number "$value" || die "$key must be a whole number, got '$value'"
      printf -v "${key,,}" '%s' "$value"
      ;;
    MOUNTS)    read -ra mounts <<< "$value" ;;
    PROCESSES) read -ra processes <<< "$value" ;;
    *) die "unknown setting '$key'" ;;
  esac
}

load_config() {
  local file=$1 key value
  [[ -r $file ]] || die "cannot read config file $file"
  while IFS='=' read -r key value || [[ -n $key ]]; do
    [[ -z $key || $key == \#* ]] && continue
    set_setting "$key" "$value"
  done < "$file"
}

# Records one result line and remembers the worst status seen so far.
record() {
  local status=$1 check=$2 detail=$3 code
  case $status in
    OK)   code=0 ;;
    WARN) code=1 ;;
    *)    code=2 ;;
  esac
  (( code > worst )) && worst=$code
  results+=("$(printf '%-6s %-20s %s' "$status" "$check" "$detail")")
}

# Prints OK, WARN or FAIL for a value where higher is worse.
grade_high() {
  local value=$1 warn=$2 fail=$3
  if (( value >= fail )); then echo FAIL
  elif (( value >= warn )); then echo WARN
  else echo OK
  fi
}

check_disk() {
  local mount=$1 used
  if ! used=$(df --output=pcent "$mount" 2>/dev/null | tail -n 1); then
    record FAIL "disk $mount" "not found"
    return
  fi
  used=${used//[ %]/}
  record "$(grade_high "$used" "$disk_warn" "$disk_fail")" "disk $mount" "${used}% used"
}

check_memory() {
  local total available pct status
  total=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
  available=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
  pct=$(( available * 100 / total ))
  if (( pct <= mem_fail )); then status=FAIL
  elif (( pct <= mem_warn )); then status=WARN
  else status=OK
  fi
  record "$status" "memory" "${pct}% available ($(( available / 1024 )) MiB)"
}

check_load() {
  local load1 cpus whole frac pct
  read -r load1 _ < /proc/loadavg
  cpus=$(nproc)
  whole=${load1%.*}
  frac=${load1#*.}
  # Work in hundredths so Bash's integer maths is enough: 1.50 becomes 150.
  pct=$(( (10#$whole * 100 + 10#$frac) / cpus ))
  record "$(grade_high "$pct" "$load_warn" "$load_fail")" "load" "$load1 on $cpus CPUs (${pct}%)"
}

check_process() {
  local name=$1 count
  count=$(pgrep -cx -- "$name" || true)
  if (( count > 0 )); then
    record OK "process $name" "$count running"
  else
    record FAIL "process $name" "not running"
  fi
}

print_table() {
  printf '%-6s %-20s %s\n' STATUS CHECK DETAIL
  printf '%s\n' "${results[@]}"
}

write_report() {
  local file=$1 tmp
  tmp=$(mktemp "$file.XXXXXX") || die "cannot write to $(dirname "$file")"
  trap 'rm -f "$tmp"' EXIT
  {
    printf 'Health check on %s at %s\n' "$(hostname)" "$(date '+%F %T')"
    print_table
  } > "$tmp"
  mv "$tmp" "$file"
  trap - EXIT
  log "report written to $file"
}

main() {
  local opt config="" extra_mounts=() extra_processes=() overrides=()

  while getopts ':c:s:d:p:o:h' opt; do
    case $opt in
      c) config=$OPTARG ;;
      s) overrides+=("$OPTARG") ;;
      d) extra_mounts+=("$OPTARG") ;;
      p) extra_processes+=("$OPTARG") ;;
      o) report_file=$OPTARG ;;
      h) usage; exit 0 ;;
      :) usage >&2; die "option -$OPTARG needs a value" ;;
      *) usage >&2; die "unknown option -$OPTARG" ;;
    esac
  done
  shift $((OPTIND - 1))
  (( $# == 0 )) || { usage >&2; die "unexpected argument '$1'"; }

  # Config file first, then -s, so the command line wins.
  [[ -n $config ]] && load_config "$config"
  local pair
  for pair in "${overrides[@]}"; do
    [[ $pair == *=* ]] || die "-s needs KEY=VALUE, got '$pair'"
    set_setting "${pair%%=*}" "${pair#*=}"
  done
  (( ${#extra_mounts[@]} > 0 )) && mounts=("${extra_mounts[@]}")
  (( ${#extra_processes[@]} > 0 )) && processes=("${extra_processes[@]}")

  local mount name
  for mount in "${mounts[@]}"; do
    check_disk "$mount"
  done
  check_memory
  check_load
  for name in "${processes[@]}"; do
    check_process "$name"
  done

  print_table
  [[ -n $report_file ]] && write_report "$report_file"
  exit "$worst"
}

main "$@"

Two lines deserve a closer look:

• printf -v "${key,,}" '%s' "$value" assigns to the variable whose name is the key in lower case, so DISK_WARN sets disk_warn. The case above it means only known names get this far.
• In write_report, the report goes to a temporary file in the same directory and is then renamed over the old one with mv. A rename within one file system is a single step, so a monitor reading the file sees the old report or the new one, never half of one. The trap removes the temporary file if the script dies before the rename.

root@lab:~/scripts# chmod +x health-check.sh
root@lab:~/scripts# shellcheck health-check.sh
root@lab:~/scripts#

root@lab:~/scripts# EDITOR=nano crontab -e

Install nano first with apt install -y nano if you have not. Add these lines, save and exit:

*/5 * * * * /root/scripts/health-check.sh -c /root/scripts/health.conf -o /var/log/health-report.txt > /dev/null 2>&1 || echo "exit $? at $(date)" >> /var/log/health-alerts.log
* * * * * /root/scripts/health-check.sh -c /root/scripts/health.conf -o /var/log/health-report.txt > /dev/null 2>&1 || echo "exit $? at $(date)" >> /var/log/health-alerts.log

Wait until the minute after next, then look at what cron produced:

root@lab:~/scripts# date
Sun Oct 25 09:20:15 UTC 2026
root@lab:~/scripts# cat /var/log/health-report.txt
Health check on lab at 2026-10-25 09:20:01
STATUS CHECK                DETAIL
OK     disk /               5% used
OK     disk /root/scripts   61% used
OK     memory               92% available (7269 MiB)
OK     load                 0.31 on 8 CPUs (3%)
OK     process sleep        1 running
OK     process cron         2 running
root@lab:~/scripts# cat /var/log/health-alerts.log
cat: /var/log/health-alerts.log: No such file or directory

Every check is OK, because cron and sleep are both running. There are two cron processes: the daemon, and the copy of it that is running this job. Stop the sleep and wait another minute:

root@lab:~/scripts# pkill -x sleep
[1]+  Terminated              sleep infinity
root@lab:~/scripts# cat /var/log/health-report.txt
Health check on lab at 2026-10-25 09:21:01
STATUS CHECK                DETAIL
OK     disk /               5% used
OK     disk /root/scripts   61% used
OK     memory               92% available (7280 MiB)
OK     load                 0.63 on 8 CPUs (7%)
FAIL   process sleep        not running
OK     process cron         2 running
root@lab:~/scripts# cat /var/log/health-alerts.log
exit 2 at Sun Oct 25 09:21:01 UTC 2026

The alert log shows exit 2, a failure. Now run crontab -e again and delete the every-minute line.