check_openshift_pv_avail

#!/bin/bash

set -e -u -o pipefail

. /usr/lib/nagios-plugins-openshift/utils

usage() {
  echo "Usage: $0 -f <path> [-w <warn>] [-c <crit>]"
  echo '          [-l [<class>,]<label>=<value> [-w <warn>] [-c <crit>] ...]'
  echo '          [-s [<class>,]<capacity> [-w <warn>] [-c <crit>] ...]'
  echo
  echo 'Options:'
  echo ' -f   Config file path'
  echo ' -w   Exit with WARNING status if fewer physical volumes than the number'
  echo '      given are available'
  echo ' -c   Exit with CRITICAL status if fewer physical volumes than the number'
  echo '      given are available'
  echo ' -l   Select using label, value and optional storage class.'
  echo ' -s   Select using volume capacity and optional storage class.'
  echo
  echo 'Volumes selected using the "-l" or "-s" options are not considered for the'
  echo 'generic or later selection-specific limits. The first selector matching a'
  echo 'volume applies.'
  echo
  echo 'The "-l" and "-s" selectors support specifying a storage class name, i.e.'
  echo '"-s bulk,5Gi". By default the limits following apply to all storage'
  echo 'classes. Use the empty string (i.e. "-s ,1Gi") to select the default'
  echo 'storage class'
  echo
  echo 'Examples:'
  echo " $0 -w 10"
  echo "    Warn if fewer than 10 volumes are available"
  echo " $0 -c 33 -l customer=foocorp -w 50"
  echo '    Count volumes with the label "customer" set to "foocorp" separately and'
  echo '    warn if less than 50 are available. Fail if less than 33 volumes without'
  echo '    said label value are available.'
  echo " $0 -w 10 -s 1Gi -w 3"
  echo '    Warn if fewer than 3 volumes with a capacity of 1Gi are available or if'
  echo '    fewer than 10 of other capacities are available.'
  echo " $0 -l foo=bar -w 7 -s 100Gi"
  echo '    Ignore volumes with a capacity of 100Gi (i.e. ignore them) unless they'
  echo '    have a label "foo" with value "bar". A warning is emitted if fewer than'
  echo '    7 volumes with label "foo" and value "bar" are available.'
}

opt_cfgfile=
opt_storageclass='*'
opt_label=
opt_capacity=
opt_warn=
opt_crit=

limits=()
default_limits=

push_limit() {
  local values="${opt_warn};${opt_crit}"

  if [[ -z "$opt_label" && -z "$opt_capacity" ]]; then
    if [[ "$opt_storageclass" != '*' ]]; then
      echo 'Storage class can not be given for default limits' >&2
      exit 1
    fi

    default_limits="$values"
  else
    limits+=( "${opt_storageclass};${opt_label};${opt_capacity};${values}" )
  fi
}

validate_key_value() {
  if [[ "$1" != ?*=* ]]; then
    echo "Label must be at least one character followed by an equal sign' \
      'and a value: $1" >&2
    exit "$state_critical"
  fi
}

validate_capacity() {
  local re='^[0-9]+(\.[0-9]+)?([MGTE]i?)?$'

  if ! [[ "$1" =~ $re ]]; then
    echo "Capacity must match regular expression \"${re}\": $1" >&2
    exit "$state_critical"
  fi
}

reset_limit() {
  local varname="$1" value="$2" validatefn="$3"
  local first second

  opt_storageclass='*'
  opt_label=
  opt_capacity=
  opt_warn=
  opt_crit=

  IFS=, read -r first second <<<"$value"

  if [[ -n "$second" ]]; then
    opt_storageclass="$first"
    value="$second"
  else
    value="$first"
  fi

  if [[ -n "$validatefn" ]]; then
    "$validatefn" "$value"
  fi

  eval "${varname}=\"\$value\""
}

while getopts 'hf:w:c:l:s:' opt; do
  case "$opt" in
    h)
      usage
      exit 0
      ;;
    f) opt_cfgfile="$OPTARG" ;;
    w)
      validate_integer "$OPTARG"
      opt_warn="$OPTARG"
      ;;
    c)
      validate_integer "$OPTARG"
      opt_crit="$OPTARG"
      ;;
    l)
      push_limit
      reset_limit opt_label "$OPTARG" validate_key_value
      ;;
    s)
      push_limit
      reset_limit opt_capacity "$OPTARG" validate_capacity
      ;;
    *)
      usage >&2
      exit 1
      ;;
  esac
done

push_limit

shift $((OPTIND - 1))

if [[ "$#" -gt 0 ]]; then
  usage >&2
  exit 1
fi

if [[ -z "$opt_cfgfile" ]]; then
  usage >&2
  exit 1
fi

tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' EXIT

# Capture stderr in variable and redirect stdout to file
# shellcheck disable=SC2069
if ! msg=$(run_oc "$opt_cfgfile" get --output=json persistentvolumes 2>&1 >"$tmpdir/pv.json"); then
  echo "$msg"
  exit "$state_critical"
fi

exit_status=$state_ok
output=()
metrics=()

#
# Select volumes from a v1.PersistentVolumeList
# https://docs.openshift.com/enterprise/3.2/rest_api/kubernetes_v1.html#v1-persistentvolumelist
#
# Arguments:
# - storageclass: "*" to consider volumes of all storage classes, "" for the
#     default storage class or an explicit class name
# - labelval: Empty string to select all volumes or "label=value" to select
#     volumes with given label set to given value
# - capacity: Empty string to select all capacities or value to select volumes
#     with given capacity
# - *args: Volume UIDs to exclude
#
filter_volumes() {
  local storageclass="$1"; shift
  local labelval="$1"; shift
  local capacity="$1"; shift
  local label=
  local value=
  local exclude_uid=()

  for uid; do
    # TODO: Implement proper value escaping
    exclude_uid+=( "\"$uid\"" )
  done

  jqfilter='[
    .items[] |
    . as $cur |
    select(
      (
        ($storageclass == "*") or
        ($cur.spec.storageClassName // "") == $storageclass
      ) and (
        (
          ($cur.metadata.annotations["pv.kubernetes.io/provisioned-by"] // null) or
          (($exclude_uid | index($cur.metadata.uid)) != null)
        ) | not
      )
    )
  '
  if [[ -n "$labelval" ]]; then
    IFS='=' read label value <<< "$labelval"
    jqfilter+=" | select(.metadata.labels | .[\"$label\"] == \"$value\")"
  fi
  if [[ -n "$capacity" ]]; then
    # FIXME: Don't use string comparison to compare quantities
    jqfilter+=" | select(.spec.capacity.storage == \"$capacity\")"
  fi
  jqfilter+=']'

  jq --raw-output \
    --arg storageclass "$storageclass" \
    --argjson exclude_uid "[$(join_args ', ' ${exclude_uid[@]+"${exclude_uid[@]}"})]" \
    "$jqfilter"
}

#
# Extract UIDs from a list of objects with v1.ObjectMeta
# https://docs.openshift.com/enterprise/3.2/rest_api/kubernetes_v1.html#v1-objectmeta
#
get_obj_uids() {
  jq -r '.[] | .metadata.uid | @text'
}

#
# Count volumes (v1.PersistentVolume) grouped by capacity and storage class
#
calc_stats() {
  jq -r '
    group_by(.spec.storageClassName, .spec.capacity.storage) |
    map({
      storageclass: (.[0].spec.storageClassName // ""),
      capacity: .[0].spec.capacity.storage,
      total: length,
      avail: [.[] | select(.status.phase == "Available")] | length,
    }) |
    sort_by(.storageclass, .capacity) |
    .[] |
    @sh "storageclass=\(.storageclass) capacity=\(.capacity) total=\(.total) avail=\(.avail)"
  '
}

#
# Apply user-provided limits and calculate metrics
#
check() {
  local labelval="$1" capacity="$2" warn="$3" crit="$4"
  local status scmetric scoutput

  calc_stats > "$tmpdir/stats.txt"

  while read line; do
    eval "local $line"

    used=$(( total - avail ))

    if [[ -z "$storageclass" ]]; then
      scmetric=
      scoutput=" in default class"
    else
      scmetric="${storageclass}-"
      scoutput=" in class \"${storageclass}\""
    fi

    # http://docs.icinga.org/latest/en/perfdata.html#perfdata-format
    metrics+=(
      "'${scmetric}${labelval:+${labelval//=/-}-}${capacity}'=${used};${warn};${crit};0;${total}"
      )

    if [[ -n "$crit" && "$avail" -lt "$crit" ]]; then
      status=$state_critical
    elif [[ -n "$warn" && "$avail" -lt "$warn" ]]; then
      status=$state_warning
    else
      status=$state_ok
    fi

    if [[ "$status" != $state_ok ]]; then
      output+=(
        "${labelval:+[$labelval] }$avail of $total volumes with $capacity available${scoutput}"
        )
    fi

    exit_status=$(merge_status "$exit_status" "$status")
  done < "$tmpdir/stats.txt"
}

processed=()

for line in ${limits[@]+"${limits[@]}"}; do
  IFS=';' read storageclass labelval capacity warn crit _ <<< "$line"

  # Find matching disks
  filter_volumes "$storageclass" "$labelval" "$capacity" ${processed[@]+"${processed[@]}"} \
    < "$tmpdir/pv.json" > "$tmpdir/matching.json"

  processed+=( $(get_obj_uids < "$tmpdir/matching.json") )

  check "$labelval" "$capacity" "$warn" "$crit" < "$tmpdir/matching.json"
done

# Process remaining volumes, i.e. those not explicitly requested by label value
filter_volumes '*' '' '' ${processed[@]+"${processed[@]}"} \
  < "$tmpdir/pv.json" > "$tmpdir/matching.json"

IFS=';' read warn crit _ <<< "$default_limits"
check '' '' "$warn" "$crit" < "$tmpdir/matching.json"

finish "$exit_status" \
  "$(join_args ', ' ${output[@]+"${output[@]}"})" \
  "${metrics[*]+${metrics[*]}}"

# vim: set sw=2 sts=2 et :