#!/bin/sh
#
# Dump host diagnostics
#

# TODO: Would be nice if this could be appropriately sudoed (sudone?)

if [ "$(id -u)" != "0" ]
then
    echo "This program must be run as root." 1>&2
    exit 1
fi

# ------------------------------------------------------------------------------

# Code in this section from ShellER under the Apache License, Version
# 2.0.  See https://github.com/markfeit/sheller.

# Print a horizontal line to the standard output
line()
{
    cat <<EOF
-------------------------------------------------------------------------------
EOF
}


# Print all lines of input centered for an 80-character line
center()
{
    awk '{ printf "%*s%s\n", int((80-length)/2), "", $0 }'   
}



# Print a nicely-formatted section header
section()
{
    echo
    echo
    line
    echo "$@" | tr a-z A-Z | center
    line
    echo
}


# ------------------------------------------------------------------------------

# Other Utilities

dump_url()
{
    curl -s -k "$1" 2>&1
}


LOG_LIMIT=1000

# ------------------------------------------------------------------------------

#
# Header
#

echo "pScheduler System Diagnostic Dump" | center
echo
echo "Generated for $(id -nu) @ $(hostname)" | center
echo "$(date)" | center


#
# pScheduler
#


section "Installed Tests"
dump_url 'https://localhost/pscheduler/tests?pretty&expanded=true'

section "Installed Tools"
dump_url 'https://localhost/pscheduler/tests?pretty&expanded=true'

section "Tool Configurations"

for CONFIG in $(find /etc/pscheduler/tool -type f -print)
do
    echo "$(basename ${CONFIG}):"
    echo
    cat  "${CONFIG}"
    echo
    echo
done

section "Installed Archivers"
dump_url 'https://localhost/pscheduler/archivers?pretty&expanded=true'

section "Global Archivers"

FOUND=false
for FILE in $(find %{_pscheduler_archiver_default_dir} -type f)
do
    echo
    echo "${FILE}:"
    cat "${FILE}"
    FOUND=true
done

$FOUND || echo "None installed"



section "Clock State"
dump_url 'https://localhost/pscheduler/clock?pretty'



section "Near-Term Schedule"

pscheduler schedule -PT2H +PT2H


section "New Tasks by Hour Added"

postgresql-load <<EOF
\c pscheduler
SELECT
date_trunc('hour', added) AS hour,
count(*)
FROM
task
GROUP BY hour
ORDER BY hour
;
EOF


section "New Runs by Hour Added"

postgresql-load <<EOF
\c pscheduler
SELECT
date_trunc('hour', added) AS hour,
count(*)
FROM
run
GROUP BY hour
ORDER BY hour
;
EOF


section "Run States by Hour"

postgresql-load <<EOF
\c pscheduler
SELECT
run_state.enum AS state,
date_trunc('hour', added) AS hour,
count(*)
FROM
run
JOIN run_state ON run_state.id = run.state
GROUP BY run_state.id, hour
ORDER BY run_state.id, hour
;
EOF


section pScheduler Log
PSCHEDULER_LOG=/var/log/pscheduler/pscheduler.log
echo $PSCHEDULER_LOG
if [ -r "${PSCHEDULER_LOG}" ]
then
    tail "-${LOG_LIMIT}" "${PSCHEDULER_LOG}"
else
    echo "Unable to find a readable pScheduler log."
fi



#
# Network
#


section Network Interfaces
ip address

section Resolver Configuration
cat /etc/resolv.conf

section Routing Table
ip route


section Sockets
ss -n -a


#
# System
#

section Process Table
ps -ef

section Memory
vmstat

section Storage
df -h

section "System Message Log (Last ${LOG_LIMIT} Lines)"

for LOG in /var/log/messages /var/log/syslog
do
    if [ -r "${LOG}" ]
    then
	tail "-${LOG_LIMIT}" "${LOG}"
	break
    fi
    # This makes sure LOG is empty after the last iteration if the
    # file wasn't found.
    LOG=
done

if [ -z "${LOG}" ]
then
    echo "Unable to find a readable system log."
fi


section "HTTPD Error Log (Last ${LOG_LIMIT} Lines)"
LOG="/var/log/httpd/error_log"
if [ -r "${LOG}" ]
then
    tail "-${LOG_LIMIT}" "${LOG}"
else
    echo "${LOG} is unreadable."
fi


section "PostgreSQL Connection Summary"

postgresql-load <<EOF

SELECT datname, application_name, count(*)
FROM pg_stat_activity
GROUP BY datname, application_name
ORDER BY datname, application_name
;
EOF


section "PostgreSQL Log (Last ${LOG_LIMIT} Lines)"

PG_DATA=$(postgresql-load <<EOF
\\t
\\a
SELECT setting FROM pg_settings WHERE name = 'data_directory';
EOF
)
PG_LOG="${PG_DATA}/pg_log"

if [ -d "${PG_LOG}" ]
then
    ls -rtd ${PG_DATA}/pg_log/* | xargs cat | tail "-${LOG_LIMIT}"
else
    echo "PostgreSQL logs not found in ${PG_LOG}"
fi


section DMI Table
dmidecode


section Kernel Message Log
dmesg


section Open Files
if [ -e '/.dockerenv' ]
then
    echo "Skipping this step because lsof doesn't work well under Docker."
else
    lsof
fi


section Installed Packages

if [ -e /etc/redhat-release ]
then
    rpm -qa --last | sort
elif which dpkg-query > /dev/null 2>&1
then
    dpkg-query --list
else
    echo "Don't know how to list packages in this environment."
fi
